Commit 1405f010 authored by Eloy Lafuente's avatar Eloy Lafuente
Browse files

MDL-51580 upgradelib: Delete stuff used by removed upgrade steps

This commits removes stuff from different upgradelib files, used
exclusively by the already deleted upgrade steps. Given such
exclusivity it was not needed to proceed with a 2-phase deprecation
as far as the functions were 100% internal to upgrade.

This is the list of deleted functions, all them docummented in their
corresponding upgrade.txt files:

- repository_picasa_admin_upgrade_notification();
- repository_googledocs_admin_upgrade_notification();
- repository_boxnet_admin_upgrade_notification();
- repository_alfresco_admin_security_key_notice();
- qtype_essay_convert_to_html();
- portfolio_picasa_admin_upgrade_notification();
- portfolio_googledocs_admin_upgrade_notification();
- portfolio_boxnet_admin_upgrade_notification();
- mod_book_migrate_moddata_dir_to_legacy();
- mod_book_migrate_all_areas();
- mod_book_migrate_area();
- mod_assignment_pending_upgrades_notification();
- upgrade_mysql_fix_unsigned_and_lob_columns();
- upgrade_course_completion_remove_duplicates();
- upgrade_save_orphaned_questions();
- upgrade_rename_old_backup_files_using_shortname();
- upgrade_mssql_nvarcharmax();
- upgrade_mssql_varbinarymax();
- upgrade_fix_missing_root_folders();
- upgrade_course_modules_sequences();
- upgrade_grade_item_fix_sortorder();
- upgrade_availability_item();
parent e8c82aac
......@@ -76,387 +76,6 @@ function upgrade_mysql_get_supported_tables() {
return $tables;
}
/**
* Remove all signed numbers from current database and change
* text fields to long texts - mysql only.
*/
function upgrade_mysql_fix_unsigned_and_lob_columns() {
// We are not using standard API for changes of column
// because everything 'signed'-related will be removed soon.
// If anybody already has numbers higher than signed limit the execution stops
// and tables must be fixed manually before continuing upgrade.
global $DB;
if ($DB->get_dbfamily() !== 'mysql') {
return;
}
$pbar = new progress_bar('mysqlconvertunsignedlobs', 500, true);
$prefix = $DB->get_prefix();
$tables = upgrade_mysql_get_supported_tables();
$tablecount = count($tables);
$i = 0;
foreach ($tables as $table) {
$i++;
$changes = array();
$sql = "SHOW COLUMNS FROM `{{$table}}`";
$rs = $DB->get_recordset_sql($sql);
foreach ($rs as $column) {
$column = (object)array_change_key_case((array)$column, CASE_LOWER);
if (stripos($column->type, 'unsigned') !== false) {
$maxvalue = 0;
if (preg_match('/^int/i', $column->type)) {
$maxvalue = 2147483647;
} else if (preg_match('/^medium/i', $column->type)) {
$maxvalue = 8388607;
} else if (preg_match('/^smallint/i', $column->type)) {
$maxvalue = 32767;
} else if (preg_match('/^tinyint/i', $column->type)) {
$maxvalue = 127;
}
if ($maxvalue) {
// Make sure nobody is abusing our integer ranges - moodle int sizes are in digits, not bytes!!!
$invalidcount = $DB->get_field_sql("SELECT COUNT('x') FROM `{{$table}}` WHERE `$column->field` > :maxnumber", array('maxnumber'=>$maxvalue));
if ($invalidcount) {
throw new moodle_exception('notlocalisederrormessage', 'error', new moodle_url('/admin/'), "Database table '{$table}'' contains unsigned column '{$column->field}' with $invalidcount values that are out of allowed range, upgrade can not continue.");
}
}
$type = preg_replace('/unsigned/i', 'signed', $column->type);
$notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
$default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
$autoinc = (stripos($column->extra, 'auto_increment') !== false) ? 'AUTO_INCREMENT' : '';
// Primary and unique not necessary here, change_database_structure does not add prefix.
$changes[] = "MODIFY COLUMN `$column->field` $type $notnull $default $autoinc";
} else if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text') {
$notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
$default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
// Primary, unique and inc are not supported for texts.
$changes[] = "MODIFY COLUMN `$column->field` LONGTEXT $notnull $default";
} else if ($column->type === 'tinyblob' or $column->type === 'mediumblob' or $column->type === 'blob') {
$notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
$default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
// Primary, unique and inc are not supported for blobs.
$changes[] = "MODIFY COLUMN `$column->field` LONGBLOB $notnull $default";
}
}
$rs->close();
if ($changes) {
// Set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case.
$count = $DB->count_records($table, array());
$timeout = ($count/1000)*60;
$timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout;
upgrade_set_timeout($timeout);
$sql = "ALTER TABLE `{$prefix}$table` ".implode(', ', $changes);
$DB->change_database_structure($sql);
}
$pbar->update($i, $tablecount, "Converted unsigned/lob columns in MySQL database - $i/$tablecount.");
}
}
/**
* Migrate NTEXT to NVARCHAR(MAX).
*/
function upgrade_mssql_nvarcharmax() {
global $DB;
if ($DB->get_dbfamily() !== 'mssql') {
return;
}
$pbar = new progress_bar('mssqlconvertntext', 500, true);
$prefix = $DB->get_prefix();
$tables = $DB->get_tables(false);
$tablecount = count($tables);
$i = 0;
foreach ($tables as $table) {
$i++;
$columns = array();
$sql = "SELECT column_name
FROM INFORMATION_SCHEMA.COLUMNS
WHERE table_name = '{{$table}}' AND UPPER(data_type) = 'NTEXT'";
$rs = $DB->get_recordset_sql($sql);
foreach ($rs as $column) {
$columns[] = $column->column_name;
}
$rs->close();
if ($columns) {
// Set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case.
$count = $DB->count_records($table, array());
$timeout = ($count/1000)*60;
$timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout;
upgrade_set_timeout($timeout);
$updates = array();
foreach ($columns as $column) {
// Change the definition.
$sql = "ALTER TABLE {$prefix}$table ALTER COLUMN $column NVARCHAR(MAX)";
$DB->change_database_structure($sql);
$updates[] = "$column = $column";
}
// Now force the migration of text data to new optimised storage.
$sql = "UPDATE {{$table}} SET ".implode(', ', $updates);
$DB->execute($sql);
}
$pbar->update($i, $tablecount, "Converted NTEXT to NVARCHAR(MAX) columns in MS SQL Server database - $i/$tablecount.");
}
}
/**
* Migrate IMAGE to VARBINARY(MAX).
*/
function upgrade_mssql_varbinarymax() {
global $DB;
if ($DB->get_dbfamily() !== 'mssql') {
return;
}
$pbar = new progress_bar('mssqlconvertimage', 500, true);
$prefix = $DB->get_prefix();
$tables = $DB->get_tables(false);
$tablecount = count($tables);
$i = 0;
foreach ($tables as $table) {
$i++;
$columns = array();
$sql = "SELECT column_name
FROM INFORMATION_SCHEMA.COLUMNS
WHERE table_name = '{{$table}}' AND UPPER(data_type) = 'IMAGE'";
$rs = $DB->get_recordset_sql($sql);
foreach ($rs as $column) {
$columns[] = $column->column_name;
}
$rs->close();
if ($columns) {
// Set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case.
$count = $DB->count_records($table, array());
$timeout = ($count/1000)*60;
$timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout;
upgrade_set_timeout($timeout);
foreach ($columns as $column) {
// Change the definition.
$sql = "ALTER TABLE {$prefix}$table ALTER COLUMN $column VARBINARY(MAX)";
$DB->change_database_structure($sql);
}
// Binary columns should not be used, do not waste time optimising the storage.
}
$pbar->update($i, $tablecount, "Converted IMAGE to VARBINARY(MAX) columns in MS SQL Server database - $i/$tablecount.");
}
}
/**
* This upgrade script fixes the mismatches between DB fields course_modules.section
* and course_sections.sequence. It makes sure that each module is included
* in the sequence of at least one section.
* Note that this script is different from admin/cli/fix_course_sortorder.php
* in the following ways:
* 1. It does not fix the cases when module appears several times in section(s) sequence(s) -
* it will be done automatically on the next viewing of the course.
* 2. It does not remove non-existing modules from section sequences - administrator
* has to run the CLI script to do it.
* 3. When this script finds an orphaned module it adds it to the section but makes hidden
* where CLI script does not change the visiblity specified in the course_modules table.
*/
function upgrade_course_modules_sequences() {
global $DB;
// Find all modules that point to the section which does not point back to this module.
$sequenceconcat = $DB->sql_concat("','", "s.sequence", "','");
$moduleconcat = $DB->sql_concat("'%,'", "m.id", "',%'");
$sql = "SELECT m.id, m.course, m.section, s.sequence
FROM {course_modules} m LEFT OUTER JOIN {course_sections} s
ON m.course = s.course and m.section = s.id
WHERE s.sequence IS NULL OR ($sequenceconcat NOT LIKE $moduleconcat)
ORDER BY m.course";
$rs = $DB->get_recordset_sql($sql);
$sections = null;
foreach ($rs as $cm) {
if (!isset($sections[$cm->course])) {
// Retrieve all sections for the course (only once for each corrupt course).
$sections = array($cm->course =>
$DB->get_records('course_sections', array('course' => $cm->course),
'section', 'id, section, sequence, visible'));
if (empty($sections[$cm->course])) {
// Very odd - the course has a module in it but has no sections. Create 0-section.
$newsection = array('sequence' => '', 'section' => 0, 'visible' => 1);
$newsection['id'] = $DB->insert_record('course_sections',
$newsection + array('course' => $cm->course, 'summary' => '', 'summaryformat' => FORMAT_HTML));
$sections[$cm->course] = array($newsection['id'] => (object)$newsection);
}
}
// Attempt to find the section that has this module in it's sequence.
// If there are several of them, pick the last because this is what get_fast_modinfo() does.
$sectionid = null;
foreach ($sections[$cm->course] as $section) {
if (!empty($section->sequence) && in_array($cm->id, preg_split('/,/', $section->sequence))) {
$sectionid = $section->id;
}
}
if ($sectionid) {
// Found the section. Update course_module to point to the correct section.
$params = array('id' => $cm->id, 'section' => $sectionid);
if (!$sections[$cm->course][$sectionid]->visible) {
$params['visible'] = 0;
}
$DB->update_record('course_modules', $params);
} else {
// No section in the course has this module in it's sequence.
if (isset($sections[$cm->course][$cm->section])) {
// Try to add module to the section it points to (if it is valid).
$sectionid = $cm->section;
} else {
// Section not found. Just add to the first available section.
reset($sections[$cm->course]);
$sectionid = key($sections[$cm->course]);
}
$newsequence = ltrim($sections[$cm->course][$sectionid]->sequence . ',' . $cm->id, ',');
$sections[$cm->course][$sectionid]->sequence = $newsequence;
$DB->update_record('course_sections', array('id' => $sectionid, 'sequence' => $newsequence));
// Make module invisible because it was not displayed at all before this upgrade script.
$DB->update_record('course_modules', array('id' => $cm->id, 'section' => $sectionid, 'visible' => 0, 'visibleold' => 0));
}
}
$rs->close();
unset($sections);
// Note that we don't need to reset course cache here because it is reset automatically after upgrade.
}
/**
* Updates a single item (course module or course section) to transfer the
* availability settings from the old to the new format.
*
* Note: We do not convert groupmembersonly for modules at present. If we did,
* $groupmembersonly would be set to the groupmembersonly option for the
* module. Since we don't, it will be set to 0 for modules, and 1 for sections
* if they have a grouping.
*
* @param int $groupmembersonly 1 if activity has groupmembersonly option
* @param int $groupingid Grouping id (0 = none)
* @param int $availablefrom Available from time (0 = none)
* @param int $availableuntil Available until time (0 = none)
* @param int $showavailability Show availability (1) or hide activity entirely
* @param array $availrecs Records from course_modules/sections_availability
* @param array $fieldrecs Records from course_modules/sections_avail_fields
*/
function upgrade_availability_item($groupmembersonly, $groupingid,
$availablefrom, $availableuntil, $showavailability,
array $availrecs, array $fieldrecs) {
global $CFG, $DB;
$conditions = array();
$shows = array();
// Group members only condition (if enabled).
if ($CFG->enablegroupmembersonly && $groupmembersonly) {
if ($groupingid) {
$conditions[] = '{"type":"grouping"' .
($groupingid ? ',"id":' . $groupingid : '') . '}';
} else {
// No grouping specified, so allow any group.
$conditions[] = '{"type":"group"}';
}
// Group members only condition was not displayed to students.
$shows[] = 'false';
// In the unlikely event that the site had enablegroupmembers only
// but NOT enableavailability, we need to turn this on now.
if (!$CFG->enableavailability) {
set_config('enableavailability', 1);
}
}
// Date conditions.
if ($availablefrom) {
$conditions[] = '{"type":"date","d":">=","t":' . $availablefrom . '}';
$shows[] = $showavailability ? 'true' : 'false';
}
if ($availableuntil) {
$conditions[] = '{"type":"date","d":"<","t":' . $availableuntil . '}';
// Until dates never showed to students.
$shows[] = 'false';
}
// Conditions from _availability table.
foreach ($availrecs as $rec) {
if (!empty($rec->sourcecmid)) {
// Completion condition.
$conditions[] = '{"type":"completion","cm":' . $rec->sourcecmid .
',"e":' . $rec->requiredcompletion . '}';
} else {
// Grade condition.
$minmax = '';
if (!empty($rec->grademin)) {
$minmax .= ',"min":' . sprintf('%.5f', $rec->grademin);
}
if (!empty($rec->grademax)) {
$minmax .= ',"max":' . sprintf('%.5f', $rec->grademax);
}
$conditions[] = '{"type":"grade","id":' . $rec->gradeitemid . $minmax . '}';
}
$shows[] = $showavailability ? 'true' : 'false';
}
// Conditions from _fields table.
foreach ($fieldrecs as $rec) {
if (isset($rec->userfield)) {
// Standard field.
$fieldbit = ',"sf":' . json_encode($rec->userfield);
} else {
// Custom field.
$fieldbit = ',"cf":' . json_encode($rec->shortname);
}
// Value is not included for certain operators.
switch($rec->operator) {
case 'isempty':
case 'isnotempty':
$valuebit = '';
break;
default:
$valuebit = ',"v":' . json_encode($rec->value);
break;
}
$conditions[] = '{"type":"profile","op":"' . $rec->operator . '"' .
$fieldbit . $valuebit . '}';
$shows[] = $showavailability ? 'true' : 'false';
}
// If there are some conditions, set them into database.
if ($conditions) {
return '{"op":"&","showc":[' . implode(',', $shows) . '],' .
'"c":[' . implode(',', $conditions) . ']}';
} else {
return null;
}
}
/**
* Using data for a single course-module that has groupmembersonly enabled,
* returns the new availability value that incorporates the correct
......
......@@ -43,80 +43,6 @@ class core_upgradelib_testcase extends advanced_testcase {
$this->assertFalse(upgrade_stale_php_files_present());
}
/**
* Test the {@link upgrade_grade_item_fix_sortorder() function with
* faked duplicate sortorder data.
*/
public function test_upgrade_grade_item_fix_sortorder() {
global $DB;
$this->resetAfterTest(true);
// The purpose of this test is to make sure that after upgrade script
// there is no duplicates in the field grade_items.sortorder (for each course)
// and the result of query "SELECT id FROM grade_items WHERE courseid=? ORDER BY sortorder, id" does not change.
$sequencesql = 'SELECT id FROM {grade_items} WHERE courseid=? ORDER BY sortorder, id';
// Each set is used for filling the db with fake data and will be representing the result of query:
// "SELECT sortorder from {grade_items} WHERE courseid=? ORDER BY id".
$testsets = array(
// Items that need no action.
array(1,2,3),
array(5,6,7),
array(7,6,1,3,2,5),
// Items with sortorder duplicates
array(1,2,2,3,3,4,5),
// Only one sortorder duplicate.
array(1,1),
array(3,3),
// Non-sequential sortorders with one or multiple duplicates.
array(3,3,7,5,6,6,9,10,8,3),
array(7,7,3),
array(3,4,5,3,5,4,7,1)
);
$origsequences = array();
// Generate the data and remember the initial sequence or items.
foreach ($testsets as $testset) {
$course = $this->getDataGenerator()->create_course();
foreach ($testset as $sortorder) {
$this->insert_fake_grade_item_sortorder($course->id, $sortorder);
}
$DB->get_records('grade_items');
$origsequences[$course->id] = $DB->get_fieldset_sql($sequencesql, array($course->id));
}
$duplicatedetectionsql = "SELECT courseid, sortorder
FROM {grade_items}
GROUP BY courseid, sortorder
HAVING COUNT(id) > 1";
// Verify there are duplicates before we start the fix.
$dupes = $DB->record_exists_sql($duplicatedetectionsql);
$this->assertTrue($dupes);
// Do the work.
upgrade_grade_item_fix_sortorder();
// Verify that no duplicates are left in the database.
$dupes = $DB->record_exists_sql($duplicatedetectionsql);
$this->assertFalse($dupes);
// Verify that sequences are exactly the same as they were before upgrade script.
$idx = 0;
foreach ($origsequences as $courseid => $origsequence) {
if (count(($testsets[$idx])) == count(array_unique($testsets[$idx]))) {
// If there were no duplicates for this course verify that sortorders are not modified.
$newsortorders = $DB->get_fieldset_sql("SELECT sortorder from {grade_items} WHERE courseid=? ORDER BY id", array($courseid));
$this->assertEquals($testsets[$idx], $newsortorders);
}
$newsequence = $DB->get_fieldset_sql($sequencesql, array($courseid));
$this->assertEquals($origsequence, $newsequence,
"Sequences do not match for test set $idx : ".join(',', $testsets[$idx]));
$idx++;
}
}
/**
* Populate some fake grade items into the database with specified
* sortorder and course id.
......@@ -150,57 +76,6 @@ class core_upgradelib_testcase extends advanced_testcase {
return $DB->get_record('grade_items', array('id' => $item->id));
}
public function test_upgrade_fix_missing_root_folders() {
global $DB, $SITE;
$this->resetAfterTest(true);
// Setup some broken data...
// Create two resources (and associated file areas).
$this->setAdminUser();
$resource1 = $this->getDataGenerator()->get_plugin_generator('mod_resource')
->create_instance(array('course' => $SITE->id));
$resource2 = $this->getDataGenerator()->get_plugin_generator('mod_resource')
->create_instance(array('course' => $SITE->id));
// Delete the folder record of resource1 to simulate broken data.
$context = context_module::instance($resource1->cmid);
$selectargs = array('contextid' => $context->id,
'component' => 'mod_resource',
'filearea' => 'content',
'itemid' => 0);
// Verify file records exist.
$areafilecount = $DB->count_records('files', $selectargs);
$this->assertNotEmpty($areafilecount);
// Delete the folder record.
$folderrecord = $selectargs;
$folderrecord['filepath'] = '/';
$folderrecord['filename'] = '.';
// Get previous folder record.
$oldrecord = $DB->get_record('files', $folderrecord);
$DB->delete_records('files', $folderrecord);
// Verify the folder record has been removed.
$newareafilecount = $DB->count_records('files', $selectargs);
$this->assertSame($newareafilecount, $areafilecount - 1);
$this->assertFalse($DB->record_exists('files', $folderrecord));
// Run the upgrade step!
upgrade_fix_missing_root_folders();
// Verify the folder record has been restored.
$newareafilecount = $DB->count_records('files', $selectargs);
$this->assertSame($newareafilecount, $areafilecount);
$newrecord = $DB->get_record('files', $folderrecord, '*', MUST_EXIST);
// Verify the hash is correctly created.
$this->assertSame($oldrecord->pathnamehash, $newrecord->pathnamehash);
}
public function test_upgrade_fix_missing_root_folders_draft() {
global $DB, $SITE;
......@@ -245,115 +120,6 @@ class core_upgradelib_testcase extends advanced_testcase {
$this->assertEquals($originalhash, $newhash);
}
/**
* Tests the upgrade of an individual course-module or section from the
* old to new availability system. (This test does not use the database
* so it can run any time.)
*/
public function test_upgrade_availability_item() {
global $CFG;
$this->resetAfterTest();
// This function is in the other upgradelib.
require_once($CFG->libdir . '/db/upgradelib.php');
// Groupmembersonly (or nothing). Show option on but ignored.
// Note: This $CFG option doesn't exist any more but we are testing the
// upgrade function so it did exist then...
$CFG->enablegroupmembersonly = 0;
$this->assertNull(
upgrade_availability_item(1, 0, 0, 0, 1, array(), array()));
$CFG->enablegroupmembersonly = 1;
$this->assertNull(
upgrade_availability_item(0, 0, 0, 0, 1, array(), array()));
$this->assertEquals(
'{"op":"&","showc":[false],"c":[{"type":"group"}]}',
upgrade_availability_item(1, 0, 0, 0, 1, array(), array()));
$this->assertEquals(
'{"op":"&","showc":[false],"c":[{"type":"grouping","id":4}]}',
upgrade_availability_item(1, 4, 0, 0, 1, array(), array()));
// Dates (with show/hide options - until date always hides).
$this->assertEquals(
'{"op":"&","showc":[true],"c":[{"type":"date","d":">=","t":996}]}',
upgrade_availability_item(0, 0, 996, 0, 1, array(), array()));
$this->assertEquals(
'{"op":"&","showc":[false],"c":[{"type":"date","d":">=","t":997}]}',
upgrade_availability_item(0, 0, 997, 0, 0, array(), array()));
$this->assertEquals(
'{"op":"&","showc":[false],"c":[{"type":"date","d":"<","t":998}]}',
upgrade_availability_item(0, 0, 0, 998, 1, array(), array()));
$this->assertEquals(
'{"op":"&","showc":[true,false],"c":[' .
'{"type":"date","d":">=","t":995},{"type":"date","d":"<","t":999}]}',
upgrade_availability_item(0, 0, 995, 999, 1, array(), array()));
// Grade (show option works as normal).
$availrec = (object)array(
'sourcecmid' => null, 'requiredcompletion' => null,
'gradeitemid' => 13, 'grademin' => null, 'grademax' => null);
$this->assertEquals(
'{"op":"&","showc":[true],"c":[{"type":"grade","id":13}]}',
upgrade_availability_item(0, 0, 0, 0, 1, array($availrec), array()));
$availrec->grademin = 4.1;
$this->assertEquals(
'{"op":"&","showc":[false],"c":[{"type":"grade","id":13,"min":4.10000}]}',
upgrade_availability_item(0, 0, 0, 0, 0, array($availrec), array()));
$availrec->grademax = 9.9;
$this->assertEquals(
'{"op":"&","showc":[true],"c":[{"type":"grade","id":13,"min":4.10000,"max":9.90000}]}',
upgrade_availability_item(0, 0, 0, 0, 1, array($availrec), array()));
$availrec->grademin = null;
$this->assertEquals(
'{"op":"&","showc":[true],"c":[{"type":"grade","id":13,"max":9.90000}]}',
upgrade_availability_item(0, 0, 0, 0, 1, array($availrec), array()));
// Completion (show option normal).
$availrec->grademax = null;
$availrec->gradeitemid = null;
$availrec->sourcecmid = 666;
$availrec->requiredcompletion = 1;
$this->assertEquals(
'{"op":"&","showc":[true],"c":[{"type":"completion","cm":666,"e":1}]}',
upgrade_availability_item(0, 0, 0, 0, 1, array($availrec), array()));
$this->assertEquals(
'{"op":"&","showc":[false],"c":[{"type":"completion","cm":666,"e":1}]}',
upgrade_availability_item(0, 0, 0, 0, 0, array($availrec), array()));
// Profile conditions (custom/standard field, values/not, show option normal).