Commit aa5b7056 authored by David Monllaó's avatar David Monllaó
Browse files

MDL-66004 mlbackend_python: Package installed on a separate server

parent 9528b1ff
......@@ -175,6 +175,13 @@ if ($hassiteconfig) {
$plugin->load_settings($ADMIN, 'antivirussettings', $hassiteconfig);
}
// Machine learning backend plugins.
$ADMIN->add('modules', new admin_category('mlbackendsettings', new lang_string('mlbackendsettings', 'admin')));
$plugins = core_plugin_manager::instance()->get_plugins_of_type('mlbackend');
foreach ($plugins as $plugin) {
$plugin->load_settings($ADMIN, 'mlbackendsettings', $hassiteconfig);
}
/// License types
$ADMIN->add('modules', new admin_category('licensesettings', new lang_string('licenses')));
$temp = new admin_settingpage('managelicenses', new lang_string('managelicenses', 'admin'));
......
......@@ -48,7 +48,7 @@ class manager {
/**
* @var \core_analytics\predictor[]
*/
protected static $predictionprocessors = null;
protected static $predictionprocessors = [];
/**
* @var \core_analytics\local\target\base[]
......@@ -213,6 +213,14 @@ class manager {
return $predictionprocessors;
}
/**
* Resets the cached prediction processors.
* @return null
*/
public static function reset_prediction_processors() {
self::$predictionprocessors = [];
}
/**
* Returns the name of the provided predictions processor.
*
......
......@@ -539,7 +539,7 @@ class model {
debugging('Prediction processor ' . $predictorname . ' is not ready to be used. Model ' .
$this->model->id . ' could not be deleted.');
} else {
$predictor->delete_output_dir($this->get_output_dir(array(), true));
$predictor->delete_output_dir($this->get_output_dir(array(), true), $this->get_unique_id());
}
$DB->delete_records('analytics_models', array('id' => $this->model->id));
......
......@@ -70,8 +70,9 @@ interface predictor {
* can only be named 'execution', 'evaluation' or 'testing'.
*
* @param string $modeloutputdir The model directory id (parent of all model versions subdirectories).
* @param string $uniqueid
* @return null
*/
public function delete_output_dir($modeloutputdir);
public function delete_output_dir($modeloutputdir, $uniqueid);
}
......@@ -17,6 +17,14 @@
/**
* Unit tests for evaluation, training and prediction.
*
* NOTE: in order to execute this test using a separate server for the
* python ML backend you need to define these variables in your config.php file:
*
* define('TEST_MLBACKEND_PYTHON_HOST', '127.0.0.1');
* define('TEST_MLBACKEND_PYTHON_PORT', 5000);
* define('TEST_MLBACKEND_PYTHON_USERNAME', 'default');
* define('TEST_MLBACKEND_PYTHON_PASSWORD', 'sshhhh');
*
* @package core_analytics
* @copyright 2017 David Monllaó {@link http://www.davidmonllao.com}
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
......@@ -109,12 +117,18 @@ class core_analytics_prediction_testcase extends advanced_testcase {
* @param int $predictedrangeindex
* @param int $nranges
* @param string $predictionsprocessorclass
* @param array $forcedconfig
* @return void
*/
public function test_ml_training_and_prediction($timesplittingid, $predictedrangeindex, $nranges, $predictionsprocessorclass) {
public function test_ml_training_and_prediction($timesplittingid, $predictedrangeindex, $nranges, $predictionsprocessorclass,
$forcedconfig) {
global $DB;
$this->resetAfterTest(true);
$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);
$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');
......@@ -122,13 +136,8 @@ class core_analytics_prediction_testcase extends advanced_testcase {
$ncourses = 10;
$this->generate_courses($ncourses);
// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}
$model = $this->add_perfect_model();
$model->update(true, false, $timesplittingid, get_class($predictionsprocessor));
// No samples trained yet.
......@@ -250,6 +259,17 @@ class core_analytics_prediction_testcase extends advanced_testcase {
$this->assertCount(2, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
\core_analytics\dataset_manager::UNLABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
// Confirm that the files associated to the model are deleted on clear and on delete. The ML backend deletion
// processes will be triggered by these actions and any exception there would result in a failed test.
$model->clear();
$this->assertEquals(0, $DB->count_records('analytics_used_files',
array('modelid' => $model->get_id(), 'action' => 'trained')));
$this->assertCount(0, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
\core_analytics\dataset_manager::LABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
$this->assertCount(0, $fs->get_directory_files(\context_system::instance()->id, 'analytics',
\core_analytics\dataset_manager::UNLABELLED_FILEAREA, $model->get_id(), '/analysable/', true, false));
$model->delete();
set_config('enabled_stores', '', 'tool_log');
get_log_manager(true);
}
......@@ -273,11 +293,15 @@ class core_analytics_prediction_testcase extends advanced_testcase {
* test_ml_export_import
*
* @param string $predictionsprocessorclass The class name
* @param array $forcedconfig
* @dataProvider provider_ml_processors
*/
public function test_ml_export_import($predictionsprocessorclass) {
public function test_ml_export_import($predictionsprocessorclass, $forcedconfig) {
$this->resetAfterTest(true);
$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);
$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');
......@@ -285,13 +309,8 @@ class core_analytics_prediction_testcase extends advanced_testcase {
$ncourses = 10;
$this->generate_courses($ncourses);
// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}
$model = $this->add_perfect_model();
$model->update(true, false, '\core\analytics\time_splitting\quarters', get_class($predictionsprocessor));
$model->train();
......@@ -355,15 +374,14 @@ class core_analytics_prediction_testcase extends advanced_testcase {
* @param int $nsamples
* @param int $classes
* @param string $predictionsprocessorclass
* @param array $forcedconfig
* @return void
*/
public function test_ml_classifiers_return($success, $nsamples, $classes, $predictionsprocessorclass) {
public function test_ml_classifiers_return($success, $nsamples, $classes, $predictionsprocessorclass, $forcedconfig) {
$this->resetAfterTest();
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}
$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);
if ($nsamples % count($classes) != 0) {
throw new \coding_exception('The number of samples should be divisible by the number of classes');
......@@ -396,7 +414,7 @@ class core_analytics_prediction_testcase extends advanced_testcase {
// Training should work correctly if at least 1 sample of each class is included.
$dir = make_request_directory();
$result = $predictionsprocessor->train_classification('whatever', $dataset, $dir);
$result = $predictionsprocessor->train_classification('whatever' . microtime(), $dataset, $dir);
switch ($success) {
case 'yes':
......@@ -441,16 +459,19 @@ class core_analytics_prediction_testcase extends advanced_testcase {
* @dataProvider provider_test_multi_classifier
* @param string $timesplittingid
* @param string $predictionsprocessorclass
* @param array|null $forcedconfig
* @throws coding_exception
* @throws moodle_exception
*/
public function test_ml_multi_classifier($timesplittingid, $predictionsprocessorclass) {
public function test_ml_multi_classifier($timesplittingid, $predictionsprocessorclass, $forcedconfig) {
global $DB;
$this->resetAfterTest(true);
$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');
$this->set_forced_config($forcedconfig);
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
......@@ -483,6 +504,9 @@ class core_analytics_prediction_testcase extends advanced_testcase {
// The range index is not important here, both ranges prediction will be the same.
$this->assertEquals($correct[$sampleid], $predictiondata->prediction);
}
set_config('enabled_stores', '', 'tool_log');
get_log_manager(true);
}
/**
......@@ -508,10 +532,16 @@ class core_analytics_prediction_testcase extends advanced_testcase {
* @param int $ncourses
* @param array $expected
* @param string $predictionsprocessorclass
* @param array $forcedconfig
* @return void
*/
public function test_ml_evaluation_configuration($modelquality, $ncourses, $expected, $predictionsprocessorclass) {
public function test_ml_evaluation_configuration($modelquality, $ncourses, $expected, $predictionsprocessorclass,
$forcedconfig) {
$this->resetAfterTest(true);
$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);
$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');
......@@ -530,12 +560,6 @@ class core_analytics_prediction_testcase extends advanced_testcase {
// Generate training data.
$this->generate_courses($ncourses);
// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}
$model->update(false, false, false, get_class($predictionsprocessor));
$results = $model->evaluate();
......@@ -563,10 +587,15 @@ class core_analytics_prediction_testcase extends advanced_testcase {
* @coversNothing
* @dataProvider provider_ml_processors
* @param string $predictionsprocessorclass
* @param array $forcedconfig
* @return null
*/
public function test_ml_evaluation_trained_model($predictionsprocessorclass) {
public function test_ml_evaluation_trained_model($predictionsprocessorclass, $forcedconfig) {
$this->resetAfterTest(true);
$this->set_forced_config($forcedconfig);
$predictionsprocessor = $this->is_predictions_processor_ready($predictionsprocessorclass);
$this->setAdminuser();
set_config('enabled_stores', 'logstore_standard', 'tool_log');
......@@ -575,12 +604,6 @@ class core_analytics_prediction_testcase extends advanced_testcase {
// Generate training data.
$this->generate_courses(50);
// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
if ($predictionsprocessor->is_ready() !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready.');
}
$model->update(true, false, '\\core\\analytics\\time_splitting\\quarters', get_class($predictionsprocessor));
$model->train();
......@@ -824,6 +847,41 @@ class core_analytics_prediction_testcase extends advanced_testcase {
}
}
/**
* Forces some configuration values.
*
* @param array $forcedconfig
*/
protected function set_forced_config($forcedconfig) {
\core_analytics\manager::reset_prediction_processors();
if (empty($forcedconfig)) {
return;
}
foreach ($forcedconfig as $pluginname => $pluginconfig) {
foreach ($pluginconfig as $name => $value) {
set_config($name, $value, $pluginname);
}
}
}
/**
* Is the provided processor ready using the current configuration in the site?
*
* @param string $predictionsprocessorclass
* @return \core_analytics\predictor
*/
protected function is_predictions_processor_ready(string $predictionsprocessorclass) {
// We repeat the test for all prediction processors.
$predictionsprocessor = \core_analytics\manager::get_predictions_processor($predictionsprocessorclass, false);
$ready = $predictionsprocessor->is_ready();
if ($ready !== true) {
$this->markTestSkipped('Skipping ' . $predictionsprocessorclass . ' as the predictor is not ready: ' . $ready);
}
return $predictionsprocessor;
}
/**
* add_prediction_processors
*
......@@ -834,12 +892,32 @@ class core_analytics_prediction_testcase extends advanced_testcase {
$return = array();
// We need to test all system prediction processors.
if (defined('TEST_MLBACKEND_PYTHON_HOST') && defined('TEST_MLBACKEND_PYTHON_PORT')
&& defined('TEST_MLBACKEND_PYTHON_USERNAME') && defined('TEST_MLBACKEND_PYTHON_USERNAME')) {
$testpythonserver = true;
}
// We need to test all prediction processors in the system.
$predictionprocessors = \core_analytics\manager::get_all_prediction_processors();
foreach ($predictionprocessors as $classfullname => $unused) {
foreach ($predictionprocessors as $classfullname => $predictionsprocessor) {
foreach ($cases as $key => $case) {
$newkey = $key . '-' . $classfullname;
$return[$newkey] = $case + array('predictionsprocessorclass' => $classfullname);
$return[$key . '-' . $classfullname] = $case + ['predictionsprocessor' => $classfullname, 'forcedconfig' => null];
}
if ($predictionsprocessor instanceof \mlbackend_python\processor && !empty($testpythonserver)) {
// We also want to test the python processor using the server.
foreach ($cases as $key => $case) {
// We want the configuration to be forced during the test as things like importing models create new
// instances of ML backend processors during the process.
$forcedconfig = ['mlbackend_python' => ['useserver' => true, 'host' => TEST_MLBACKEND_PYTHON_HOST,
'port' => TEST_MLBACKEND_PYTHON_PORT, 'secure' => false, 'username' => TEST_MLBACKEND_PYTHON_USERNAME,
'password' => TEST_MLBACKEND_PYTHON_PASSWORD]];
$casekey = $key . '-' . $classfullname . '-server';
$return[$casekey] = $case + ['predictionsprocessor' => $classfullname, 'forcedconfig' => $forcedconfig];
}
}
}
......
......@@ -15,6 +15,7 @@ information provided here is intended especially for developers.
* A new \core_analytics\local\time_splitting\past_periodic abstract class has been added. Time-splitting
methods extending \core_analytics\local\time_splitting\periodic directly should be extending past_periodic
now. 'periodic' can still be directly extended by implementing get_next_range and get_first_start methods.
* \core_analytics\predictor::delete_output_dir has a new 2nd parameter, $uniquemodelid.
=== 3.7 ===
......
......@@ -790,6 +790,7 @@ $string['minpasswordlower'] = 'Lowercase letters';
$string['minpasswordnonalphanum'] = 'Non-alphanumeric characters';
$string['minpasswordupper'] = 'Uppercase letters';
$string['misc'] = 'Miscellaneous';
$string['mlbackendsettings'] = 'Machine learning backend settings';
$string['mnetrestore_extusers'] = '<strong>Note:</strong> This backup file contains remote Moodle Network user accounts which will be restored as part of the process.';
$string['mnetrestore_extusers_admin'] = '<strong>Note:</strong> This backup file seems to come from a different Moodle installation and contains remote Moodle Network user accounts. The restore process will try to match the Moodle Network hosts for all created users. Those not matching will be automatically switched to internal authentication (instead of mnet one). The restore log will inform you about that.';
$string['mnetrestore_extusers_mismatch'] = '<strong>Note:</strong> This backup file apparently originates from a different Moodle installation and contains remote Moodle Network user accounts that may fail to restore. This operation is unsupported. If you are certain that it was created on this Moodle installation, or you can ensure that all the needed Moodle Network Hosts are configured, you may want to still try the restore.';
......
......@@ -50,6 +50,39 @@ class mlbackend extends base {
* @return null|string node name or null if plugin does not create settings node (default)
*/
public function get_settings_section_name() {
return 'mlbackendsetting' . $this->name;
return 'mlbackendsettings' . $this->name;
}
/**
* Load the global settings for a particular availability plugin (if there are any)
*
* @param \part_of_admin_tree $adminroot
* @param string $parentnodename
* @param bool $hassiteconfig
* @return void
*/
public function load_settings(\part_of_admin_tree $adminroot, $parentnodename, $hassiteconfig) {
global $CFG, $USER, $DB, $OUTPUT, $PAGE; // In case settings.php wants to refer to them.
$ADMIN = $adminroot; // May be used in settings.php.
$plugininfo = $this; // Also can be used inside settings.php.
if (!$this->is_installed_and_upgraded()) {
return;
}
if (!$hassiteconfig) {
return;
}
$section = $this->get_settings_section_name();
$settings = null;
if (file_exists($this->full_path('settings.php'))) {
$settings = new \admin_settingpage($section, $this->displayname, 'moodle/site:config', $this->is_enabled() === false);
include($this->full_path('settings.php')); // This may also set $settings to null.
}
if ($settings) {
$ADMIN->add($parentnodename, $settings);
}
}
}
......@@ -88,9 +88,10 @@ class processor implements \core_analytics\classifier, \core_analytics\regressor
* Delete the output directory.
*
* @param string $modeloutputdir
* @param string $uniqueid
* @return null
*/
public function delete_output_dir($modeloutputdir) {
public function delete_output_dir($modeloutputdir, $uniqueid) {
remove_dir($modeloutputdir);
}
......
This diff is collapsed.
......@@ -22,9 +22,25 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
$string['errornoconfigdata'] = 'The server configuration is not complete.';
$string['errorserver'] = 'Server error {$a}';
$string['host'] = 'Host';
$string['hostdesc'] = 'Host';
$string['packageinstalledshouldbe'] = '"moodlemlbackend" python package should be updated. The required version is "{$a->required}" and the installed version is "{$a->installed}"';
$string['packageinstalledtoohigh'] = '"moodlemlbackend" python package is not compatible with this Moodle version. The required version is "{$a->required}" or higher as long as it is API-compatible. The installed version "{$a->installed}" is too high.';
$string['pluginname'] = 'Python machine learning backend';
$string['port'] = 'Port';
$string['portdesc'] = 'Port';
$string['privacy:metadata'] = 'The Python machine learning backend plugin does not store any personal data.';
$string['pythonpackagenotinstalled'] = '"moodlemlbackend" python package is not installed or there is a problem with it. Please execute "{$a}" from command line interface for more info';
$string['pythonpathnotdefined'] = 'The path to your executable Python binary has not been defined. Please visit "{$a}" to set it.';
$string['serversettingsinfo'] = 'Tick "Use a server" setting to show the server settings.';
$string['username'] = 'Username';
$string['usernamedesc'] = 'String of characters used as a username to communicate between your Moodle server and the python server';
$string['password'] = 'Password';
$string['passworddesc'] = 'String of characters used as a password to communicate between your Moodle server and the python server';
$string['secure'] = 'Use HTTPS';
$string['securedesc'] = 'Whether to use HTTP or HTTPS';
$string['useserver'] = 'Use a server';
$string['useserverdesc'] = 'The machine learning backend python package is not installed in the web server but in a different server.';
$string['tensorboardinfo'] = 'Launch TensorBoard from command line by typing tensorboard --logdir=\'{$a}\' in your web server.';
\ No newline at end of file
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Administration settings definitions for mlbackend_python.
*
* @package mlbackend_python
* @copyright 2019 David Monllaó
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
defined('MOODLE_INTERNAL') || die();
if ($ADMIN->fulltree) {
$info = $OUTPUT->notification(get_string('serversettingsinfo', 'mlbackend_python'), 'info');
$settings->add(new admin_setting_heading('mlbackend_python/serversettingsinfo', '', $info));
$settings->add(new admin_setting_configcheckbox('mlbackend_python/useserver', get_string('useserver', 'mlbackend_python'),
get_string('useserverdesc', 'mlbackend_python'), 0));
$settings->add(new admin_setting_configtext('mlbackend_python/host', get_string('host', 'mlbackend_python'),
get_string('host', 'mlbackend_python'), '', PARAM_HOST));
$settings->hide_if('mlbackend_python/host', 'mlbackend_python/useserver', 'neq', '1');
$settings->add(new admin_setting_configtext('mlbackend_python/port', get_string('port', 'mlbackend_python'),
get_string('port', 'mlbackend_python'), '', PARAM_INT));
$settings->hide_if('mlbackend_python/port', 'mlbackend_python/useserver', 'neq', '1');
$settings->add(new admin_setting_configcheckbox('mlbackend_python/secure', get_string('secure', 'mlbackend_python'),
get_string('securedesc', 'mlbackend_python'), 0));
$settings->hide_if('mlbackend_python/secure', 'mlbackend_python/useserver', 'neq', '1');
$settings->add(new admin_setting_configtext('mlbackend_python/username', get_string('username', 'mlbackend_python'),
get_string('usernamedesc', 'mlbackend_python'), 'default', PARAM_ALPHANUMEXT));
$settings->hide_if('mlbackend_python/username', 'mlbackend_python/useserver', 'neq', '1');
$settings->add(new admin_setting_configtext('mlbackend_python/password', get_string('password', 'mlbackend_python'),
get_string('passworddesc', 'mlbackend_python'), '', PARAM_ALPHANUMEXT));
$settings->hide_if('mlbackend_python/password', 'mlbackend_python/useserver', 'neq', '1');
}
\ No newline at end of file
......@@ -24,6 +24,6 @@
defined('MOODLE_INTERNAL') || die();
$plugin->version = 2019052000; // The current plugin version (Date: YYYYMMDDXX).
$plugin->version = 2019052001; // The current plugin version (Date: YYYYMMDDXX).
$plugin->requires = 2019051100; // Requires this Moodle version.
$plugin->component = 'mlbackend_python'; // Full name of the plugin (used for diagnostics).
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment