Skip to content
GitLab
Commit a3b1172d authored by Dan Poltawski's avatar Dan Poltawski
Browse files

Merge branch 'MDL-50602-master' of https://github.com/StudiUM/moodle

parents df15a718 e10c9acc
Hide whitespace changes
Inline Side-by-side
admin/settings/courses.php
View file @ a3b1172d
......@@ -226,7 +226,8 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) {
);
$temp->add(new admin_setting_configselect('backup/backup_auto_storage', new lang_string('automatedstorage', 'backup'), new lang_string('automatedstoragehelp', 'backup'), 0, $storageoptions));
$temp->add(new admin_setting_special_backup_auto_destination());
$keepoptoins = array(
$maxkeptoptions = array(
0 => new lang_string('all'), 1 => '1',
2 => '2',
5 => '5',
......@@ -240,7 +241,44 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) {
300 => '300',
400 => '400',
500 => '500');
$temp->add(new admin_setting_configselect('backup/backup_auto_keep', new lang_string('keep'), new lang_string('backupkeephelp'), 1, $keepoptoins));
$temp->add(new admin_setting_configselect('backup/backup_auto_max_kept', new lang_string('automatedmaxkept', 'backup'),
new lang_string('automatedmaxkepthelp', 'backup'), 1, $maxkeptoptions));
$automateddeletedaysoptions = array(
0 => new lang_string('never'),
1000 => new lang_string('numdays', '', 1000),
365 => new lang_string('numdays', '', 365),
180 => new lang_string('numdays', '', 180),
150 => new lang_string('numdays', '', 150),
120 => new lang_string('numdays', '', 120),
90 => new lang_string('numdays', '', 90),
60 => new lang_string('numdays', '', 60),
35 => new lang_string('numdays', '', 35),
10 => new lang_string('numdays', '', 10),
5 => new lang_string('numdays', '', 5),
2 => new lang_string('numdays', '', 2)
);
$temp->add(new admin_setting_configselect('backup/backup_auto_delete_days', new lang_string('automateddeletedays', 'backup'),
'', 0, $automateddeletedaysoptions));
$minkeptoptions = array(
0 => new lang_string('none'),
1 => '1',
2 => '2',
5 => '5',
10 => '10',
20 => '20',
30 => '30',
40 => '40',
50 => '50',
100 => '100',
200 => '200',
300 => '300',
400 => '400'
);
$temp->add(new admin_setting_configselect('backup/backup_auto_min_kept', new lang_string('automatedminkept', 'backup'),
new lang_string('automatedminkepthelp', 'backup'), 0, $minkeptoptions));
$temp->add(new admin_setting_configcheckbox('backup/backup_shortname', new lang_string('backup_shortname', 'admin'), new lang_string('backup_shortnamehelp', 'admin'), 0));
$temp->add(new admin_setting_configcheckbox('backup/backup_auto_skip_hidden', new lang_string('skiphidden', 'backup'), new lang_string('skiphiddenhelp', 'backup'), 1));
$temp->add(new admin_setting_configselect('backup/backup_auto_skip_modif_days', new lang_string('skipmodifdays', 'backup'), new lang_string('skipmodifdayshelp', 'backup'), 30, array(
......
backup/upgrade.txt
View file @ a3b1172d
This files describes API changes in /backup/*,
information provided here is intended especially for developers.
=== 3.0 ===
* The backup_auto_keep setting, in automated backups configuration, is now
renamed to backup_auto_max_kept as part of a rationalise of naming (see MDL-50602)
=== 2.6 ===
* The backup_controller_dbops::create_temptable_from_real_table()
......
backup/util/helper/backup_cron_helper.class.php
View file @ a3b1172d
......@@ -61,6 +61,13 @@ abstract class backup_cron_automated_helper {
const AUTO_BACKUP_ENABLED = 1;
const AUTO_BACKUP_MANUAL = 2;
/** Automated backup storage in course backup filearea */
const STORAGE_COURSE = 0;
/** Automated backup storage in specified directory */
const STORAGE_DIRECTORY = 1;
/** Automated backup storage in course backup filearea and specified directory */
const STORAGE_COURSE_AND_DIRECTORY = 2;
/**
* Runs the automated backups if required
*
......@@ -174,42 +181,42 @@ abstract class backup_cron_automated_helper {
$backupcourse->nextstarttime = $nextstarttime;
$DB->update_record('backup_courses', $backupcourse);
mtrace('Skipping ' . $course->fullname . ' (Not scheduled for backup until ' . $showtime . ')');
} else if ($skipped) { // Must have been skipped for a reason.
$backupcourse->laststatus = self::BACKUP_STATUS_SKIPPED;
$backupcourse->nextstarttime = $nextstarttime;
$DB->update_record('backup_courses', $backupcourse);
mtrace('Skipping ' . $course->fullname . ' (' . $skippedmessage . ')');
mtrace('Backup of \'' . $course->fullname . '\' is scheduled on ' . $showtime);
} else {
// Backup every non-skipped courses.
mtrace('Backing up '.$course->fullname.'...');
if ($skipped) { // Must have been skipped for a reason.
$backupcourse->laststatus = self::BACKUP_STATUS_SKIPPED;
$backupcourse->nextstarttime = $nextstarttime;
$DB->update_record('backup_courses', $backupcourse);
mtrace('Skipping ' . $course->fullname . ' (' . $skippedmessage . ')');
mtrace('Backup of \'' . $course->fullname . '\' is scheduled on ' . $showtime);
} else {
// Backup every non-skipped courses.
mtrace('Backing up '.$course->fullname.'...');
// We have to send an email because we have included at least one backup.
$emailpending = true;
// We have to send an email because we have included at least one backup.
$emailpending = true;
// Only make the backup if laststatus isn't 2-UNFINISHED (uncontrolled error).
if ($backupcourse->laststatus != self::BACKUP_STATUS_UNFINISHED) {
// Set laststarttime.
$starttime = time();
// Only make the backup if laststatus isn't 2-UNFINISHED (uncontrolled error).
if ($backupcourse->laststatus != self::BACKUP_STATUS_UNFINISHED) {
// Set laststarttime.
$starttime = time();
$backupcourse->laststarttime = time();
$backupcourse->laststatus = self::BACKUP_STATUS_UNFINISHED;
$DB->update_record('backup_courses', $backupcourse);
$backupcourse->laststarttime = time();
$backupcourse->laststatus = self::BACKUP_STATUS_UNFINISHED;
$DB->update_record('backup_courses', $backupcourse);
$backupcourse->laststatus = backup_cron_automated_helper::launch_automated_backup($course, $backupcourse->laststarttime, $admin->id);
$backupcourse->lastendtime = time();
$backupcourse->nextstarttime = $nextstarttime;
$backupcourse->laststatus = self::launch_automated_backup($course, $backupcourse->laststarttime,
$admin->id);
$backupcourse->lastendtime = time();
$backupcourse->nextstarttime = $nextstarttime;
$DB->update_record('backup_courses', $backupcourse);
$DB->update_record('backup_courses', $backupcourse);
if ($backupcourse->laststatus === self::BACKUP_STATUS_OK) {
// Clean up any excess course backups now that we have
// taken a successful backup.
$removedcount = backup_cron_automated_helper::remove_excess_backups($course);
mtrace("complete - next execution: $showtime");
}
}
mtrace("complete - next execution: $showtime");
// Remove excess backups.
$removedcount = self::remove_excess_backups($course, $now);
}
}
$rs->close();
......@@ -537,98 +544,177 @@ abstract class backup_cron_automated_helper {
}
/**
* Removes excess backups from the external system and the local file system.
* Removes excess backups from a specified course.
*
* The number of backups keep comes from $config->backup_auto_keep.
*
* @param stdClass $course object
* @return bool
* @param stdClass $course Course object
* @param int $now Starting time of the process
* @return bool Whether or not backups is being removed
*/
public static function remove_excess_backups($course) {
public static function remove_excess_backups($course, $now = null) {
$config = get_config('backup');
$keep = (int)$config->backup_auto_keep;
$storage = $config->backup_auto_storage;
$dir = $config->backup_auto_destination;
$maxkept = (int)$config->backup_auto_max_kept;
$storage = $config->backup_auto_storage;
$deletedays = (int)$config->backup_auto_delete_days;
if ($keep == 0) {
// Means keep all backup files.
if ($maxkept == 0 && $deletedays == 0) {
// Means keep all backup files and never delete backup after x days.
return true;
}
if (!file_exists($dir) || !is_dir($dir) || !is_writable($dir)) {
$dir = null;
if (!isset($now)) {
$now = time();
}
// Clean up excess backups in the course backup filearea.
if ($storage == 0 || $storage == 2) {
$fs = get_file_storage();
$context = context_course::instance($course->id);
$component = 'backup';
$filearea = 'automated';
$itemid = 0;
$files = array();
// Store all the matching files into timemodified => stored_file array.
foreach ($fs->get_area_files($context->id, $component, $filearea, $itemid) as $file) {
$files[$file->get_timemodified()] = $file;
$deletedcoursebackups = false;
if ($storage == self::STORAGE_COURSE || $storage == self::STORAGE_COURSE_AND_DIRECTORY) {
$deletedcoursebackups = self::remove_excess_backups_from_course($course, $now);
}
// Clean up excess backups in the specified external directory.
$deleteddirectorybackups = false;
if ($storage == self::STORAGE_DIRECTORY || $storage == self::STORAGE_COURSE_AND_DIRECTORY) {
$deleteddirectorybackups = self::remove_excess_backups_from_directory($course, $now);
}
if ($deletedcoursebackups || $deleteddirectorybackups) {
return true;
} else {
return false;
}
}
/**
* Removes excess backups in the course backup filearea from a specified course.
*
* @param stdClass $course Course object
* @param int $now Starting time of the process
* @return bool Whether or not backups are being removed
*/
protected static function remove_excess_backups_from_course($course, $now) {
$fs = get_file_storage();
$context = context_course::instance($course->id);
$component = 'backup';
$filearea = 'automated';
$itemid = 0;
$backupfiles = array();
$backupfilesarea = $fs->get_area_files($context->id, $component, $filearea, $itemid, 'timemodified DESC', false);
// Store all the matching files into timemodified => stored_file array.
foreach ($backupfilesarea as $backupfile) {
$backupfiles[$backupfile->get_timemodified()] = $backupfile;
}
$backupstodelete = self::get_backups_to_delete($backupfiles, $now);
if ($backupstodelete) {
foreach ($backupstodelete as $backuptodelete) {
$backuptodelete->delete();
}
if (count($files) <= $keep) {
// There are less matching files than the desired number to keep there is nothing to clean up.
return 0;
mtrace('Deleted ' . count($backupstodelete) . ' old backup file(s) from the automated filearea');
return true;
} else {
return false;
}
}
/**
* Removes excess backups in the specified external directory from a specified course.
*
* @param stdClass $course Course object
* @param int $now Starting time of the process
* @return bool Whether or not backups are being removed
*/
protected static function remove_excess_backups_from_directory($course, $now) {
$config = get_config('backup');
$dir = $config->backup_auto_destination;
$isnotvaliddir = !file_exists($dir) || !is_dir($dir) || !is_writable($dir);
if ($isnotvaliddir) {
mtrace('Error: ' . $dir . ' does not appear to be a valid directory');
return false;
}
// Calculate backup filename regex, ignoring the date/time/info parts that can be
// variable, depending of languages, formats and automated backup settings.
$filename = backup::FORMAT_MOODLE . '-' . backup::TYPE_1COURSE . '-' . $course->id . '-';
$regex = '#' . preg_quote($filename, '#') . '.*\.mbz$#';
// Store all the matching files into filename => timemodified array.
$backupfiles = array();
foreach (scandir($dir) as $backupfile) {
// Skip files not matching the naming convention.
if (!preg_match($regex, $backupfile)) {
continue;
}
// Sort by keys descending (newer to older filemodified).
krsort($files);
$remove = array_splice($files, $keep);
foreach ($remove as $file) {
$file->delete();
// Read the information contained in the backup itself.
try {
$bcinfo = backup_general_helper::get_backup_information_from_mbz($dir . '/' . $backupfile);
} catch (backup_helper_exception $e) {
mtrace('Error: ' . $backupfile . ' does not appear to be a valid backup (' . $e->errorcode . ')');
continue;
}
//mtrace('Removed '.count($remove).' old backup file(s) from the automated filearea');
}
// Clean up excess backups in the specified external directory.
if (!empty($dir) && ($storage == 1 || $storage == 2)) {
// Calculate backup filename regex, ignoring the date/time/info parts that can be
// variable, depending of languages, formats and automated backup settings.
$filename = backup::FORMAT_MOODLE . '-' . backup::TYPE_1COURSE . '-' . $course->id . '-';
$regex = '#' . preg_quote($filename, '#') . '.*\.mbz$#';
// Store all the matching files into filename => timemodified array.
$files = array();
foreach (scandir($dir) as $file) {
// Skip files not matching the naming convention.
if (!preg_match($regex, $file, $matches)) {
continue;
}
// Make sure this backup concerns the course and site we are looking for.
if ($bcinfo->format === backup::FORMAT_MOODLE &&
$bcinfo->type === backup::TYPE_1COURSE &&
$bcinfo->original_course_id == $course->id &&
backup_general_helper::backup_is_samesite($bcinfo)) {
$backupfiles[$bcinfo->backup_date] = $backupfile;
}
}
// Read the information contained in the backup itself.
try {
$bcinfo = backup_general_helper::get_backup_information_from_mbz($dir . '/' . $file);
} catch (backup_helper_exception $e) {
mtrace('Error: ' . $file . ' does not appear to be a valid backup (' . $e->errorcode . ')');
continue;
}
$backupstodelete = self::get_backups_to_delete($backupfiles, $now);
if ($backupstodelete) {
foreach ($backupstodelete as $backuptodelete) {
unlink($dir . '/' . $backuptodelete);
}
mtrace('Deleted ' . count($backupstodelete) . ' old backup file(s) from external directory');
return true;
} else {
return false;
}
}
// Make sure this backup concerns the course and site we are looking for.
if ($bcinfo->format === backup::FORMAT_MOODLE &&
$bcinfo->type === backup::TYPE_1COURSE &&
$bcinfo->original_course_id == $course->id &&
backup_general_helper::backup_is_samesite($bcinfo)) {
$files[$file] = $bcinfo->backup_date;
/**
* Get the list of backup files to delete depending on the automated backup settings.
*
* @param array $backupfiles Existing backup files
* @param int $now Starting time of the process
* @return array Backup files to delete
*/
protected static function get_backups_to_delete($backupfiles, $now) {
$config = get_config('backup');
$maxkept = (int)$config->backup_auto_max_kept;
$deletedays = (int)$config->backup_auto_delete_days;
$minkept = (int)$config->backup_auto_min_kept;
// Sort by keys descending (newer to older filemodified).
krsort($backupfiles);
$tokeep = $maxkept;
if ($deletedays > 0) {
$deletedayssecs = $deletedays * DAYSECS;
$tokeep = 0;
$backupfileskeys = array_keys($backupfiles);
foreach ($backupfileskeys as $timemodified) {
$mustdeletebackup = $timemodified < ($now - $deletedayssecs);
if ($mustdeletebackup || $tokeep >= $maxkept) {
break;
}
$tokeep++;
}
if (count($files) <= $keep) {
// There are less matching files than the desired number to keep there is nothing to clean up.
return 0;
}
// Sort by values descending (newer to older filemodified).
arsort($files);
$remove = array_splice($files, $keep);
foreach (array_keys($remove) as $file) {
unlink($dir . '/' . $file);
if ($tokeep < $minkept) {
$tokeep = $minkept;
}
//mtrace('Removed '.count($remove).' old backup file(s) from external directory');
}
return true;
if (count($backupfiles) <= $tokeep) {
// There are less or equal matching files than the desired number to keep, there is nothing to clean up.
return false;
} else {
$backupstodelete = array_splice($backupfiles, $tokeep);
return $backupstodelete;
}
}
/**
......
backup/util/helper/tests/cronhelper_test.php
View file @ a3b1172d
......@@ -244,4 +244,100 @@ class backup_cron_helper_testcase extends advanced_testcase {
$next = backup_cron_automated_helper::calculate_next_automated_backup($timezone, $now);
$this->assertEquals(date('w-20:00'), date('w-H:i', $next));
}
/**
* Test {@link backup_cron_automated_helper::get_backups_to_delete}.
*/
public function test_get_backups_to_delete() {
$this->resetAfterTest();
// Active only backup_auto_max_kept config to 2 days.
set_config('backup_auto_max_kept', '2', 'backup');
set_config('backup_auto_delete_days', '0', 'backup');
set_config('backup_auto_min_kept', '0', 'backup');
// No backups to delete.
$backupfiles = array(
'1000000000' => 'file1.mbz',
'1000432000' => 'file3.mbz'
);
$deletedbackups = testable_backup_cron_automated_helper::testable_get_backups_to_delete($backupfiles, 1000432000);
$this->assertFalse($deletedbackups);
// Older backup to delete.
$backupfiles['1000172800'] = 'file2.mbz';
$deletedbackups = testable_backup_cron_automated_helper::testable_get_backups_to_delete($backupfiles, 1000432000);
$this->assertEquals(1, count($deletedbackups));
$this->assertArrayHasKey('1000000000', $backupfiles);
$this->assertEquals('file1.mbz', $backupfiles['1000000000']);
// Activate backup_auto_max_kept to 5 days and backup_auto_delete_days to 10 days.
set_config('backup_auto_max_kept', '5', 'backup');
set_config('backup_auto_delete_days', '10', 'backup');
set_config('backup_auto_min_kept', '0', 'backup');
// No backups to delete. Timestamp is 1000000000 + 10 days.
$backupfiles['1000432001'] = 'file4.mbz';
$backupfiles['1000864000'] = 'file5.mbz';
$deletedbackups = testable_backup_cron_automated_helper::testable_get_backups_to_delete($backupfiles, 1000864000);
$this->assertFalse($deletedbackups);
// One old backup to delete. Timestamp is 1000000000 + 10 days + 1 second.
$deletedbackups = testable_backup_cron_automated_helper::testable_get_backups_to_delete($backupfiles, 1000864001);
$this->assertEquals(1, count($deletedbackups));
$this->assertArrayHasKey('1000000000', $backupfiles);
$this->assertEquals('file1.mbz', $backupfiles['1000000000']);
// Two old backups to delete. Timestamp is 1000000000 + 12 days + 1 second.
$deletedbackups = testable_backup_cron_automated_helper::testable_get_backups_to_delete($backupfiles, 1001036801);
$this->assertEquals(2, count($deletedbackups));
$this->assertArrayHasKey('1000000000', $backupfiles);
$this->assertEquals('file1.mbz', $backupfiles['1000000000']);
$this->assertArrayHasKey('1000172800', $backupfiles);
$this->assertEquals('file2.mbz', $backupfiles['1000172800']);
// Activate backup_auto_max_kept to 5 days, backup_auto_delete_days to 10 days and backup_auto_min_kept to 2.
set_config('backup_auto_max_kept', '5', 'backup');
set_config('backup_auto_delete_days', '10', 'backup');
set_config('backup_auto_min_kept', '2', 'backup');
// Three instead of four old backups are deleted. Timestamp is 1000000000 + 16 days.
$deletedbackups = testable_backup_cron_automated_helper::testable_get_backups_to_delete($backupfiles, 1001382400);
$this->assertEquals(3, count($deletedbackups));
$this->assertArrayHasKey('1000000000', $backupfiles);
$this->assertEquals('file1.mbz', $backupfiles['1000000000']);
$this->assertArrayHasKey('1000172800', $backupfiles);
$this->assertEquals('file2.mbz', $backupfiles['1000172800']);
$this->assertArrayHasKey('1000432000', $backupfiles);
$this->assertEquals('file3.mbz', $backupfiles['1000432000']);
// Three instead of all five backups are deleted. Timestamp is 1000000000 + 60 days.
$deletedbackups = testable_backup_cron_automated_helper::testable_get_backups_to_delete($backupfiles, 1005184000);
$this->assertEquals(3, count($deletedbackups));
$this->assertArrayHasKey('1000000000', $backupfiles);
$this->assertEquals('file1.mbz', $backupfiles['1000000000']);
$this->assertArrayHasKey('1000172800', $backupfiles);
$this->assertEquals('file2.mbz', $backupfiles['1000172800']);
$this->assertArrayHasKey('1000432000', $backupfiles);
$this->assertEquals('file3.mbz', $backupfiles['1000432000']);
}
}
/**
* Provides access to protected methods we want to explicitly test
*
* @copyright 2015 Jean-Philippe Gaudreau <jp.gaudreau@umontreal.ca>
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class testable_backup_cron_automated_helper extends backup_cron_automated_helper {
/**
* Provides access to protected method get_backups_to_remove.
*
* @param array $backupfiles Existing backup files
* @param int $now Starting time of the process
* @return array Backup files to remove
*/
public static function testable_get_backups_to_delete($backupfiles, $now) {
return parent::get_backups_to_delete($backupfiles, $now);
}
}
lang/en/backup.php
View file @ a3b1172d
......@@ -30,6 +30,11 @@ $string['automatedbackupschedule'] = 'Schedule';
$string['automatedbackupschedulehelp'] = 'Choose which days of the week to perform automated backups.';
$string['automatedbackupsinactive'] = 'Automated backups haven\'t been enabled by the site admin';
$string['automatedbackupstatus'] = 'Automated backup status';
$string['automateddeletedays'] = 'Delete backups older than';
$string['automatedmaxkept'] = 'Maximum number of backups kept';
$string['automatedmaxkepthelp'] = 'This specifies the maximum number of recent automated backups to be kept for each course. Older backups will be deleted automatically.';
$string['automatedminkept'] = 'Minimum number of backups kept';
$string['automatedminkepthelp'] = 'If backups older than a specified number of days are deleted, it can happen that an inactive course ends up with no backup. To prevent this, a minimum number of backups kept should be specified.';
$string['automatedsetup'] = 'Automated backup setup';
$string['automatedsettings'] = 'Automated backup settings';
$string['automatedstorage'] = 'Automated backup storage';
......
lang/en/moodle.php
View file @ a3b1172d
......@@ -186,7 +186,6 @@ $string['backupfromthissite'] = 'Backup was made on this site?';
$string['backupgradebookhistoryhelp'] = 'If enabled then gradebook history will be included in automated backups. Note that grade history must not be disabled in server settings (disablegradehistory) in order for this to work';
$string['backupincludemoduleshelp'] = 'Choose whether you want to include course modules, with or without user data, in automated backups';
$string['backupincludemoduleuserdatahelp'] = 'Choose whether you want to include module user data in automated backups.';
$string['backupkeephelp'] = 'How many recent backups for each course do you want to keep? (older ones will be deleted automatically)';
$string['backuplogdetailed'] = 'Detailed execution log';
$string['backuploglaststatus'] = 'Last execution log';
$string['backupmissinguserinfoperms'] = 'Note: This backup contains no user data. Exercise and Workshop activities will not be included in the backup, since these modules are not compatible with this type of backup.';
......
lib/db/upgrade.php
View file @ a3b1172d
......@@ -4572,5 +4572,17 @@ function xmldb_main_upgrade($oldversion) {
upgrade_main_savepoint(true, 2015092200.00);
}
if ($oldversion < 2015092900.00) {
// Rename backup_auto_keep setting to backup_auto_max_kept.
$keep = get_config('backup', 'backup_auto_keep');
if ($keep !== false) {
set_config('backup_auto_max_kept', $keep, 'backup');
unset_config('backup_auto_keep', 'backup');
}
// Main savepoint reached.
upgrade_main_savepoint(true, 2015092900.00);
}
return true;
}
version.php
View file @ a3b1172d
......@@ -29,7 +29,7 @@
defined('MOODLE_INTERNAL') || die();
$version = 2015092800.00; // YYYYMMDD = weekly release date of this DEV branch.
$version = 2015092900.00; // YYYYMMDD = weekly release date of this DEV branch.
// RR = release increments - 00 in DEV branches.
// .XX = incremental changes.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment