Merge branch 'wip-MDL-48228-master-v2' of git://github.com/abgreeve/moodle

.travis.yml

@@ -134,11 +134,13 @@ before_script:
           sed -i \
             -e "s%= 'pgsql'%= 'mysqli'%" \
             -e "s%= 'username'%= 'travis'%" \
+            -e "s%=> 'utf8mb4_unicode_ci'%=> 'utf8mb4_bin'%" \
             config.php;
 
           mysql -u root -e 'SET GLOBAL innodb_file_format=barracuda;' ;
           mysql -u root -e 'SET GLOBAL innodb_file_per_table=ON;' ;
-          mysql -e 'CREATE DATABASE travis_ci_test DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_bin;' ;
+          mysql -u root -e 'SET GLOBAL innodb_large_prefix=ON;' ;
+          mysql -e 'CREATE DATABASE travis_ci_test DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_bin;' ;
         fi
       fi
 

admin/cli/mysql_collation.php

@@ -64,6 +64,71 @@ if (!empty($options['collation'])) {
         cli_error("Error: collation '$collation' is not available on this server!");
     }
 
+    $collationinfo = explode('_', $collation);
+    $charset = reset($collationinfo);
+
+    $engine = strtolower($DB->get_dbengine());
+
+    // Do checks for utf8mb4.
+    if (strpos($collation, 'utf8mb4') === 0) {
+        // Do we have the right engine?
+        if ($engine !== 'innodb' && $engine !== 'xtradb') {
+            cli_error("Error: '$collation' requires InnoDB or XtraDB set as the engine.");
+        }
+        // Are we using Barracuda?
+        if ($DB->get_row_format() != 'Barracuda') {
+            // Try setting it here.
+            try {
+                $DB->execute("SET GLOBAL innodb_file_format=Barracuda");
+            } catch (dml_exception $e) {
+                cli_error("Error: '$collation' requires the file format to be set to Barracuda.
+                        An attempt was made to change the format, but it failed. Please try doing this manually.");
+            }
+            echo "GLOBAL SETTING: innodb_file_format changed to Barracuda\n";
+        }
+        // Is one file per table being used?
+        if (!$DB->is_file_per_table_enabled()) {
+            try {
+                $DB->execute("SET GLOBAL innodb_file_per_table=1");
+            } catch (dml_exception $e) {
+                cli_error("Error: '$collation' requires the setting 'innodb_file_per_table' be set to 'ON'.
+                        An attempt was made to change the format, but it failed. Please try doing this manually.");
+            }
+            echo "GLOBAL SETTING: innodb_file_per_table changed to 1\n";
+        }
+        // Is large prefix set?
+        if (!$DB->is_large_prefix_enabled()) {
+            try {
+                $DB->execute("SET GLOBAL innodb_large_prefix=1");
+            } catch (dml_exception $e) {
+                cli_error("Error: '$collation' requires the setting 'innodb_large_prefix' be set to 'ON'.
+                        An attempt was made to change the format, but it failed. Please try doing this manually.");
+            }
+            echo "GLOBAL SETTING: innodb_large_prefix changed to 1\n";
+        }
+    }
+
+    $sql = "SHOW VARIABLES LIKE 'collation_database'";
+    if (!$dbcollation = $DB->get_record_sql($sql)) {
+        cli_error("Error: Could not access collation information on the database.");
+    }
+    $sql = "SHOW VARIABLES LIKE 'character_set_database'";
+    if (!$dbcharset = $DB->get_record_sql($sql)) {
+        cli_error("Error: Could not access character set information on the database.");
+    }
+    if ($dbcollation->value !== $collation || $dbcharset->value !== $charset) {
+        // Try to convert the DB.
+        echo "Converting database to '$collation' for $CFG->wwwroot:\n";
+        $sql = "ALTER DATABASE $CFG->dbname DEFAULT CHARACTER SET $charset DEFAULT COLLATE = $collation";
+        try {
+            $DB->change_database_structure($sql);
+        } catch (exception $e) {
+            cli_error("Error: Tried to alter the database with no success. Please try manually changing the database
+                    to the new collation and character set and then run this script again.");
+        }
+        echo "DATABASE CONVERTED\n";
+    }
+
     echo "Converting tables and columns to '$collation' for $CFG->wwwroot:\n";
     $prefix = $DB->get_prefix();
     $prefix = str_replace('_', '\\_', $prefix);
@@ -80,7 +145,7 @@ if (!empty($options['collation'])) {
             $skipped++;
 
         } else {
-            $DB->change_database_structure("ALTER TABLE $table->name DEFAULT COLLATE = $collation");
+            $DB->change_database_structure("ALTER TABLE $table->name DEFAULT CHARACTER SET $charset DEFAULT COLLATE = $collation");
             echo "CONVERTED\n";
             $converted++;
         }
@@ -96,18 +161,32 @@ if (!empty($options['collation'])) {
                 continue;
             }
 
+            // Check for utf8mb4 collation.
+            $rowformat = $DB->get_row_format_sql($engine, $collation);
+
             if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text' or $column->type === 'longtext') {
                 $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
                 $default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
                 // primary, unique and inc are not supported for texts
-                $sql = "ALTER TABLE $table->name MODIFY COLUMN $column->field $column->type COLLATE $collation $notnull $default";
+                $sql = "ALTER TABLE $table->name
+                        MODIFY COLUMN $column->field $column->type
+                        CHARACTER SET $charset
+                        COLLATE $collation $notnull $default";
                 $DB->change_database_structure($sql);
 
             } else if (strpos($column->type, 'varchar') === 0) {
                 $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
                 $default = !is_null($column->default) ? "DEFAULT '$column->default'" : '';
-                // primary, unique and inc are not supported for texts
-                $sql = "ALTER TABLE $table->name MODIFY COLUMN $column->field $column->type COLLATE $collation $notnull $default";
+
+                if ($rowformat != '') {
+                    $sql = "ALTER TABLE $table->name $rowformat";
+                    $DB->change_database_structure($sql);
+                }
+
+                $sql = "ALTER TABLE $table->name
+                        MODIFY COLUMN $column->field $column->type
+                        CHARACTER SET $charset
+                        COLLATE $collation $notnull $default";
                 $DB->change_database_structure($sql);
             } else {
                 echo "ERROR (unknown column type: $column->type)\n";
@@ -180,7 +259,9 @@ function mysql_get_collations() {
     global $DB;
 
     $collations = array();
-    $sql = "SHOW COLLATION WHERE Collation LIKE 'utf8\_%' AND Charset = 'utf8'";
+    $sql = "SHOW COLLATION
+            WHERE Collation LIKE 'utf8\_%' AND Charset = 'utf8'
+               OR Collation LIKE 'utf8mb4\_%' AND Charset = 'utf8mb4'";
     $rs = $DB->get_recordset_sql($sql);
     foreach ($rs as $collation) {
         $collations[$collation->collation] = $collation->collation;

admin/environment.xml

@@ -1723,6 +1723,21 @@
           <ON_CHECK message="libcurlwarning" />
         </FEEDBACK>
       </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_file_format" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddbfileformat" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_file_per_table" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddbfilepertable" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_large_prefix" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddblargeprefix" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
     </CUSTOM_CHECKS>
   </MOODLE>
   <MOODLE version="3.2" requires="2.7">
@@ -1873,6 +1888,21 @@
           <ON_CHECK message="libcurlwarning" />
         </FEEDBACK>
       </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_file_format" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddbfileformat" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_file_per_table" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddbfilepertable" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_large_prefix" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddblargeprefix" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
     </CUSTOM_CHECKS>
   </MOODLE>
 </COMPATIBILITY_MATRIX>

config-dist.php

@@ -63,6 +63,13 @@ $CFG->dboptions = array(
                                 // support advanced options on connection.
                                 // If you set those in the database then
                                 // the advanced settings will not be sent.
+    'dbcollation' => 'utf8mb4_unicode_ci', // MySQL has partial and full UTF-8
+                                // support. If you wish to use partial UTF-8
+                                // (three bytes) then set this option to
+                                // 'utf8_unicode_ci', otherwise this option
+                                // can be removed for MySQL (by default it will
+                                // use 'utf8mb4_unicode_ci'. This option should
+                                // be removed for all other databases.
 );
 
 

lang/en/admin.php

@@ -596,6 +596,7 @@ $string['htmlsettings'] = 'HTML settings';
 $string['http'] = 'HTTP';
 $string['httpsecurity'] = 'HTTP security';
 $string['hubs'] = 'Hubs';
+$string['incompleteunicodesupport'] = 'The current setup of MySQL or MariaDB is using \'utf8\'. This character set does not support four byte characters which include some emoji. Trying to use these characters will result in an error when updating a record, and any information being sent to the database will be lost. Please consider changing your settings to \'utf8mb4\'. See the documentation for full details.';
 $string['change'] = 'change';
 $string['checkboxno'] = 'No';
 $string['checkboxyes'] = 'Yes';
@@ -1133,6 +1134,9 @@ $string['unlockaccount'] = 'Unlock account';
 $string['unoconvwarning'] = 'The version of unoconv you have installed is not supported. Moodle\'s assignment grading feature requires version 0.7 or higher.';
 $string['unsettheme'] = 'Unset theme';
 $string['unsupported'] = 'Unsupported';
+$string['unsupporteddbfileformat'] = 'Your database has tables using Antelope as the file format. Full UTF-8 support in MySQL and MariaDB requires the Barracuda file format. Please convert the tables to the Barracuda file format. See the documentation <a href="https://docs.moodle.org/en/cli">Administration via command line</a> for details of a tool for converting InnoDB tables to Barracuda.';
+$string['unsupporteddbfilepertable'] = 'For full support of UTF-8 both MySQL and MariaDB require you to change your MySQL setting \'innodb_file_per_table\' to \'ON\'. See the documentation for further details.';
+$string['unsupporteddblargeprefix'] = 'For full support of UTF-8 both MySQL and MariaDB require you to change your MySQL setting \'innodb_large_prefix\' to \'ON\'. See the documentation for further details.';
 $string['unsupporteddbstorageengine'] = 'The database storage engine being used is no longer supported.';
 $string['unsupporteddbtablerowformat'] = 'Your database has tables using Antelope as the file format. You are recommended to convert the tables to the Barracuda file format. See the documentation <a href="https://docs.moodle.org/en/cli">Administration via command line</a> for details of a tool for converting InnoDB tables to Barracuda.';
 $string['unsupportedphpversion7'] = 'PHP version 7 is not supported.';

lib/ddl/mysql_sql_generator.php

@@ -218,6 +218,9 @@ class mysql_sql_generator extends sql_generator {
             }
         }
 
+        $utf8mb4rowformat = $this->mdb->get_row_format_sql($engine, $collation);
+        $rowformat = ($utf8mb4rowformat == '') ? $rowformat : $utf8mb4rowformat;
+
         $sqlarr = parent::getCreateTableSQL($xmldb_table);
 
         // This is a very nasty hack that tries to use just one query per created table
@@ -238,7 +241,7 @@ class mysql_sql_generator extends sql_generator {
                     if (strpos($collation, 'utf8_') === 0) {
                         $sql .= "\n DEFAULT CHARACTER SET utf8";
                     }
-                    $sql .= "\n DEFAULT COLLATE = $collation";
+                    $sql .= "\n DEFAULT COLLATE = $collation ";
                 }
                 if ($rowformat) {
                     $sql .= $rowformat;
@@ -326,10 +329,13 @@ class mysql_sql_generator extends sql_generator {
      * @return array of sql statements
      */
     public function getCreateTempTableSQL($xmldb_table) {
+        $engine = $this->mdb->get_dbengine();
         // Do we know collation?
         $collation = $this->mdb->get_dbcollation();
         $this->temptables->add_temptable($xmldb_table->getName());
 
+        $rowformat = $this->mdb->get_row_format_sql($engine, $collation);
+
         $sqlarr = parent::getCreateTableSQL($xmldb_table);
 
         // Let's inject the extra MySQL tweaks.
@@ -341,7 +347,7 @@ class mysql_sql_generator extends sql_generator {
                     if (strpos($collation, 'utf8_') === 0) {
                         $sqlarr[$i] .= " DEFAULT CHARACTER SET utf8";
                     }
-                    $sqlarr[$i] .= " DEFAULT COLLATE $collation";
+                    $sqlarr[$i] .= " DEFAULT COLLATE $collation $rowformat";
                 }
             }
         }

lib/ddl/tests/ddl_test.php

@@ -584,7 +584,7 @@ class core_ddl_testcase extends database_driver_testcase {
         $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
         $table->add_field('name', XMLDB_TYPE_CHAR, '30', null, null, null, null);
         $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
-        for ($i = 0; $i < 15; $i++) {
+        for ($i = 0; $i < 12; $i++) {
             $table->add_field('text'.$i, XMLDB_TYPE_CHAR, '1333', null, null, null, null);
             $data->{'text'.$i} = $text;
         }

lib/dml/mysqli_native_moodle_database.php

@@ -85,13 +85,19 @@ class mysqli_native_moodle_database extends moodle_database {
             throw new dml_connection_exception($dberr);
         }
 
-        if (isset($dboptions['dbcollation']) and strpos($dboptions['dbcollation'], 'utf8_') === 0) {
+        // Normally a check would be done before setting utf8mb4, but the database can be created
+        // before the enviroment checks are done. We'll proceed with creating the database and then do checks next.
+        $charset = 'utf8mb4';
+        if (isset($dboptions['dbcollation']) and (strpos($dboptions['dbcollation'], 'utf8_') === 0
+                || strpos($dboptions['dbcollation'], 'utf8mb4_') === 0)) {
             $collation = $dboptions['dbcollation'];
+            $collationinfo = explode('_', $dboptions['dbcollation']);
+            $charset = reset($collationinfo);
         } else {
-            $collation = 'utf8_unicode_ci';
+            $collation = 'utf8mb4_unicode_ci';
         }
 
-        $result = $conn->query("CREATE DATABASE $dbname DEFAULT CHARACTER SET utf8 DEFAULT COLLATE ".$collation);
+        $result = $conn->query("CREATE DATABASE $dbname DEFAULT CHARACTER SET $charset DEFAULT COLLATE ".$collation);
 
         $conn->close();
 
@@ -246,6 +252,7 @@ class mysqli_native_moodle_database extends moodle_database {
         }
         $result->close();
 
+
         if (!$collation) {
             // Get the default database collation, but only if using UTF-8.
             $sql = "SELECT @@collation_database";
@@ -253,7 +260,7 @@ class mysqli_native_moodle_database extends moodle_database {
             $result = $this->mysqli->query($sql);
             $this->query_end($result);
             if ($rec = $result->fetch_assoc()) {
-                if (strpos($rec['@@collation_database'], 'utf8_') === 0) {
+                if (strpos($rec['@@collation_database'], 'utf8_') === 0 || strpos($rec['@@collation_database'], 'utf8mb4_') === 0) {
                     $collation = $rec['@@collation_database'];
                 }
             }
@@ -263,7 +270,7 @@ class mysqli_native_moodle_database extends moodle_database {
         if (!$collation) {
             // We want only utf8 compatible collations.
             $collation = null;
-            $sql = "SHOW COLLATION WHERE Collation LIKE 'utf8\_%' AND Charset = 'utf8'";
+            $sql = "SHOW COLLATION WHERE Collation LIKE 'utf8mb4\_%' AND Charset = 'utf8mb4'";
             $this->query_start($sql, NULL, SQL_QUERY_AUX);
             $result = $this->mysqli->query($sql);
             $this->query_end($result);
@@ -288,17 +295,25 @@ class mysqli_native_moodle_database extends moodle_database {
      * @param string $table
      * @return string row_format name or null if not known or table does not exist.
      */
-    public function get_row_format($table) {
+    public function get_row_format($table = null) {
         $rowformat = null;
-        $table = $this->mysqli->real_escape_string($table);
-        $sql = "SELECT row_format
-                  FROM INFORMATION_SCHEMA.TABLES
-                 WHERE table_schema = DATABASE() AND table_name = '{$this->prefix}$table'";
+        if (isset($table)) {
+            $table = $this->mysqli->real_escape_string($table);
+            $sql = "SELECT row_format
+                      FROM INFORMATION_SCHEMA.TABLES
+                     WHERE table_schema = DATABASE() AND table_name = '{$this->prefix}$table'";
+        } else {
+            $sql = "SHOW VARIABLES LIKE 'innodb_file_format'";
+        }
         $this->query_start($sql, NULL, SQL_QUERY_AUX);
         $result = $this->mysqli->query($sql);
         $this->query_end($result);
         if ($rec = $result->fetch_assoc()) {
-            $rowformat = $rec['row_format'];
+            if (isset($table)) {
+                $rowformat = $rec['row_format'];
+            } else {
+                $rowformat = $rec['Value'];
+            }
         }
         $result->close();
 
@@ -329,16 +344,10 @@ class mysqli_native_moodle_database extends moodle_database {
             // Other engines are not supported, most probably not compatible.
             $this->compressedrowformatsupported = false;
 
-        } else if (!$filepertable = $this->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_per_table'")) {
+        } else if (!$this->is_file_per_table_enabled()) {
             $this->compressedrowformatsupported = false;
 
-        } else if ($filepertable->value !== 'ON') {
-            $this->compressedrowformatsupported = false;
-
-        } else if (!$fileformat = $this->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_format'")) {
-            $this->compressedrowformatsupported = false;
-
-        } else  if ($fileformat->value !== 'Barracuda') {
+        } else if ($this->get_row_format() !== 'Barracuda') {
             $this->compressedrowformatsupported = false;
 
         } else {
@@ -349,6 +358,67 @@ class mysqli_native_moodle_database extends moodle_database {
         return $this->compressedrowformatsupported;
     }
 
+    /**
+     * Check the database to see if innodb_file_per_table is on.
+     *
+     * @return bool True if on otherwise false.
+     */
+    public function is_file_per_table_enabled() {
+        if ($filepertable = $this->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_per_table'")) {
+            if ($filepertable->value == 'ON') {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Check the database to see if innodb_large_prefix is on.
+     *
+     * @return bool True if on otherwise false.
+     */
+    public function is_large_prefix_enabled() {
+        if ($largeprefix = $this->get_record_sql("SHOW VARIABLES LIKE 'innodb_large_prefix'")) {
+            if ($largeprefix->value == 'ON') {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Determine if the row format should be set to compressed, dynamic, or default.
+     *
+     * Terrible kludge. If we're using utf8mb4 AND we're using InnoDB, we need to specify row format to
+     * be either dynamic or compressed (default is compact) in order to allow for bigger indexes (MySQL
+     * errors #1709 and #1071).
+     *
+     * @param  string $engine The database engine being used. Will be looked up if not supplied.
+     * @param  string $collation The database collation to use. Will look up the current collation if not supplied.
+     * @return string An sql fragment to add to sql statements.
+     */
+    public function get_row_format_sql($engine = null, $collation = null) {
+
+        if (!isset($engine)) {
+            $engine = $this->get_dbengine();
+        }
+        $engine = strtolower($engine);
+
+        if (!isset($collation)) {
+            $collation = $this->get_dbcollation();
+        }
+
+        $rowformat = '';
+        if (($engine === 'innodb' || $engine === 'xtradb') && strpos($collation, 'utf8mb4_') === 0) {
+            if ($this->is_compressed_row_format_supported()) {
+                $rowformat = "ROW_FORMAT=Compressed";
+            } else {
+                $rowformat = "ROW_FORMAT=Dynamic";
+            }
+        }
+        return $rowformat;
+    }
+
     /**
      * Returns localised database type name
      * Note: can be used before connect()
@@ -449,11 +519,19 @@ class mysqli_native_moodle_database extends moodle_database {
             throw new dml_connection_exception($dberr);
         }
 
+        if (isset($dboptions['dbcollation'])) {
+            $collationinfo = explode('_', $dboptions['dbcollation']);
+            $this->dboptions['dbcollation'] = $dboptions['dbcollation'];
+        } else {
+            $collationinfo = explode('_', $this->get_dbcollation());
+        }
+        $charset = reset($collationinfo);
+
         // Disable logging until we are fully setup.
         $this->query_log_prevent();
 
         $this->query_start("--set_charset()", null, SQL_QUERY_AUX);
-        $this->mysqli->set_charset('utf8');
+        $this->mysqli->set_charset($charset);
         $this->query_end(true);
 
         // If available, enforce strict mode for the session. That guaranties
@@ -885,7 +963,10 @@ class mysqli_native_moodle_database extends moodle_database {
         // if config table already exists it has this collation too.
         $collation = $this->get_dbcollation();
 
-        $sql = "SHOW COLLATION WHERE Collation ='$collation' AND Charset = 'utf8'";
+        $collationinfo = explode('_', $collation);
+        $charset = reset($collationinfo);
+
+        $sql = "SHOW COLLATION WHERE Collation ='$collation' AND Charset = '$charset'";
         $this->query_start($sql, NULL, SQL_QUERY_AUX);
         $result = $this->mysqli->query($sql);
         $this->query_end($result);
@@ -1519,17 +1600,23 @@ class mysqli_native_moodle_database extends moodle_database {
 
     public function sql_equal($fieldname, $param, $casesensitive = true, $accentsensitive = true, $notequal = false) {
         $equalop = $notequal ? '<>' : '=';
+
+        $collationinfo = explode('_', $this->get_dbcollation());
+        $bincollate = reset($collationinfo) . '_bin';
+
         if ($casesensitive) {
             // Current MySQL versions do not support case sensitive and accent insensitive.
-            return "$fieldname COLLATE utf8_bin $equalop $param";
+            return "$fieldname COLLATE $bincollate $equalop $param";
         } else if ($accentsensitive) {
             // Case insensitive and accent sensitive, we can force a binary comparison once all texts are using the same case.
-            return "LOWER($fieldname) COLLATE utf8_bin $equalop LOWER($param)";
+            return "LOWER($fieldname) COLLATE $bincollate $equalop LOWER($param)";
         } else {
             // Case insensitive and accent insensitive. All collations are that way, but utf8_bin.
             $collation = '';
             if ($this->get_dbcollation() == 'utf8_bin') {
                 $collation = 'COLLATE utf8_unicode_ci';
+            } else if ($this->get_dbcollation() == 'utf8mb4_bin') {
+                $collation = 'COLLATE utf8mb4_unicode_ci';
             }
             return "$fieldname $collation $equalop $param";
         }
@@ -1555,15 +1642,18 @@ class mysqli_native_moodle_database extends moodle_database {
         }
         $escapechar = $this->mysqli->real_escape_string($escapechar); // prevents problems with C-style escapes of enclosing '\'
 
+        $collationinfo = explode('_', $this->get_dbcollation());
+        $bincollate = reset($collationinfo) . '_bin';
+
         $LIKE = $notlike ? 'NOT LIKE' : 'LIKE';
 
         if ($casesensitive) {
             // Current MySQL versions do not support case sensitive and accent insensitive.
-            return "$fieldname $LIKE $param COLLATE utf8_bin ESCAPE '$escapechar'";
+            return "$fieldname $LIKE $param COLLATE $bincollate ESCAPE '$escapechar'";
 
         } else if ($accentsensitive) {
             // Case insensitive and accent sensitive, we can force a binary comparison once all texts are using the same case.
-            return "LOWER($fieldname) $LIKE LOWER($param) COLLATE utf8_bin ESCAPE '$escapechar'";
+            return "LOWER($fieldname) $LIKE LOWER($param) COLLATE $bincollate ESCAPE '$escapechar'";
 
         } else {
             // Case insensitive and accent insensitive.
@@ -1571,6 +1661,9 @@ class mysqli_native_moodle_database extends moodle_database {
             if ($this->get_dbcollation() == 'utf8_bin') {
                 // Force a case insensitive comparison if using utf8_bin.
                 $collation = 'COLLATE utf8_unicode_ci';
+            } else if ($this->get_dbcollation() == 'utf8mb4_bin') {
+                // Force a case insensitive comparison if using utf8mb4_bin.
+                $collation = 'COLLATE utf8mb4_unicode_ci';
             }
 
             return "$fieldname $LIKE $param $collation ESCAPE '$escapechar'";

lib/dml/tests/dml_test.php

@@ -5543,6 +5543,42 @@ class core_dml_testcase extends database_driver_testcase {
             $dbman->drop_table($table);
         }
     }
+
+    /**
+     * Test that the database has full utf8 support (4 bytes).
+     */
+    public function test_four_byte_character_insertion() {
+        $DB = $this->tdb;
+
+        if ($DB->get_dbfamily() === 'mysql' && strpos($DB->get_dbcollation(), 'utf8_') === 0) {
+            $this->markTestSkipped($DB->get_name() .
+                    ' does not support 4 byte characters with only a utf8 collation.
+                    Please change to utf8mb4 for full utf8 support.');
+        }
+
+        $dbman = $this->tdb->get_manager();
+
+        $table = $this->get_test_table();
+        $tablename = $table->getName();
+
+        $table->add_field('id', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
+        $table->add_field('name', XMLDB_TYPE_CHAR, '255', null, null, null, null);
+        $table->add_field('content', XMLDB_TYPE_TEXT, 'big', null, XMLDB_NOTNULL);
+        $table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
+        $dbman->create_table($table);
+
+        $data = array(
+            'name' => 'Name with a four byte character 𠮟る',
+            'content' => 'Content with a four byte emoji 📝 memo.'
+        );
+
+        $insertid = $DB->insert_record($tablename, $data);
+        $result = $DB->get_record($tablename, array('id' => $insertid));
+        $this->assertEquals($data['name'], $result->name);
+        $this->assertEquals($data['content'], $result->content);
+
+        $dbman->drop_table($table);
+    }
 }
 
 /**

lib/upgradelib.php

@@ -2148,6 +2148,104 @@ function check_database_tables_row_format(environment_results $result) {
     return null;
 }
 
+/**
+ * This function verfies that the database has tables using InnoDB Antelope row format.
+ *
+ * @param environment_results $result
+ * @return environment_results|null updated results object, or null if no Antelope table has been found.
+ */
+function check_mysql_file_format(environment_results $result) {
+    global $DB;
+
+    if ($DB->get_dbfamily() == 'mysql') {
+        $collation = $DB->get_dbcollation();
+        $collationinfo = explode('_', $collation);
+        $charset = reset($collationinfo);
+
+        if ($charset == 'utf8mb4') {
+            if ($DB->get_row_format() !== "Barracuda") {
+                $result->setInfo('mysql_full_unicode_support#File_format');
+                $result->setStatus(false);
+                return $result;
+            }
+        }
+    }
+    return null;
+}
+
+/**
+ * This function verfies that the database has a setting of one file per table. This is required for 'utf8mb4'.
+ *
+ * @param environment_results $result
+ * @return environment_results|null updated results object, or null if innodb_file_per_table = 1.
+ */
+function check_mysql_file_per_table(environment_results $result) {
+    global $DB;
+
+    if ($DB->get_dbfamily() == 'mysql') {
+        $collation = $DB->get_dbcollation();