MDL-25500 lock: New locking framework.

This locking system is designed to be used everywhere requiring
locking in Moodle. Each use of the locking system can be configured
to use a different type of locking (or the same type with a different
configuration).

The first supported lock types are memcache, memcached, file (flock),
db (specific handlers for pg, mysql and mariadb).

config-dist.php

@@ -470,6 +470,33 @@ $CFG->admin = 'admin';
 // will be sent to supportemail.
 //      $CFG->supportuserid = -20;
 //
+// Moodle 2.7 introduces a locking api for critical tasks (e.g. cron).
+// The default locking system to use is DB locking for MySQL and Postgres, and File
+// locking for Oracle and SQLServer. If $CFG->preventfilelocking is set, then the default
+// will always be DB locking. It can be manually set to one of the lock
+// factory classes listed below, or one of your own custom classes implementing the
+// \core\lock\lock_factory interface.
+//
+//      $CFG->lock_factory = "auto";
+//
+// The list of available lock factories is:
+//
+// "\\core\\lock\\file_lock_factory" - File locking
+//      Uses lock files stored by default in the dataroot. Whether this
+//      works on clusters depends on the file system used for the dataroot.
+//
+// "\\core\\lock\\db_row_lock_factory" - DB locking based on table rows.
+//
+// "\\core\\lock\\postgres_lock_factory" - DB locking based on postgres advisory locks.
+//
+// "\\core\\lock\\mysql_lock_factory" - DB locking based on mysql lock functions.
+//
+// Settings used by the lock factories
+//
+// Location for lock files used by the File locking factory. This must exist
+// on a shared file system that supports locking.
+//      $CFG->lock_file_root = $CFG->dataroot . '/lock';
+//
 //=========================================================================
 // 7. SETTINGS FOR DEVELOPMENT SERVERS - not intended for production use!!!
 //=========================================================================

lib/classes/lock/db_record_lock_factory.php

+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * This is a db record locking factory.
+ *
+ * @package    core
+ * @category   lock
+ * @copyright  Damyon Wiese 2013
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core\lock;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * This is a db record locking factory.
+ *
+ * This lock factory uses record locks relying on sql of the form "SET XXX where YYY" and checking if the
+ * value was set. It supports timeouts, autorelease and can work on any DB. The downside - is this
+ * will always be slower than some shared memory type locking function.
+ *
+ * @package   core
+ * @category  lock
+ * @copyright Damyon Wiese 2013
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class db_record_lock_factory implements lock_factory {
+
+    /** @var moodle_database $db Hold a reference to the global $DB */
+    protected $db;
+
+    /** @var string $type Used to prefix lock keys */
+    protected $type;
+
+    /** @var array $openlocks - List of held locks - used by auto-release */
+    protected $openlocks = array();
+
+    /**
+     * Is available.
+     * @return boolean - True if this lock type is available in this environment.
+     */
+    public function is_available() {
+        return true;
+    }
+
+    /**
+     * Almighty constructor.
+     * @param string $type - Used to prefix lock keys.
+     */
+    public function __construct($type) {
+        global $DB;
+
+        $this->type = $type;
+        // Save a reference to the global $DB so it will not be released while we still have open locks.
+        $this->db = $DB;
+
+        \core_shutdown_manager::register_function(array($this, 'auto_release'));
+    }
+
+    /**
+     * Return information about the blocking behaviour of the lock type on this platform.
+     * @return boolean - True
+     */
+    public function supports_timeout() {
+        return true;
+    }
+
+    /**
+     * Will this lock type will be automatically released when a process ends.
+     *
+     * @return boolean - True (shutdown handler)
+     */
+    public function supports_auto_release() {
+        return true;
+    }
+
+    /**
+     * Multiple locks for the same resource can be held by a single process.
+     * @return boolean - False - not process specific.
+     */
+    public function supports_recursion() {
+        return false;
+    }
+
+    /**
+     * This function generates a unique token for the lock to use.
+     * It is important that this token is not soley based on time as this could lead
+     * to duplicates in a clustered environment (especially on VMs due to poor time precision).
+     */
+    protected function generate_unique_token() {
+        $uuid = '';
+
+        if (function_exists("uuid_create")) {
+            $context = null;
+            uuid_create($context);
+
+            uuid_make($context, UUID_MAKE_V4);
+            uuid_export($context, UUID_FMT_STR, $uuid);
+        } else {
+            // Fallback uuid generation based on:
+            // "http://www.php.net/manual/en/function.uniqid.php#94959".
+            $uuid = sprintf('%04x%04x-%04x-%04x-%04x-%04x%04x%04x',
+
+                // 32 bits for "time_low".
+                mt_rand(0, 0xffff), mt_rand(0, 0xffff),
+
+                // 16 bits for "time_mid".
+                mt_rand(0, 0xffff),
+
+                // 16 bits for "time_hi_and_version",
+                // four most significant bits holds version number 4.
+                mt_rand(0, 0x0fff) | 0x4000,
+
+                // 16 bits, 8 bits for "clk_seq_hi_res",
+                // 8 bits for "clk_seq_low",
+                // two most significant bits holds zero and one for variant DCE1.1.
+                mt_rand(0, 0x3fff) | 0x8000,
+
+                // 48 bits for "node".
+                mt_rand(0, 0xffff), mt_rand(0, 0xffff), mt_rand(0, 0xffff));
+        }
+        return trim($uuid);
+    }
+
+
+    /**
+     * Create and get a lock
+     * @param string $resource - The identifier for the lock. Should use frankenstyle prefix.
+     * @param int $timeout - The number of seconds to wait for a lock before giving up.
+     * @param int $maxlifetime - Unused by this lock type.
+     * @return boolean - true if a lock was obtained.
+     */
+    public function get_lock($resource, $timeout, $maxlifetime = 86400) {
+
+        $token = $this->generate_unique_token();
+        $now = time();
+        $giveuptime = $now + $timeout;
+        $expires = $now + $maxlifetime;
+
+        if (!$this->db->record_exists('lock_db', array('resourcekey' => $resource))) {
+            $record = new \stdClass();
+            $record->resourcekey = $resource;
+            $result = $this->db->insert_record('lock_db', $record);
+        }
+
+        $params = array('expires' => $expires,
+                        'token' => $token,
+                        'resourcekey' => $resource,
+                        'now' => $now);
+        $sql = 'UPDATE {lock_db}
+                   SET
+                       expires = :expires,
+                       owner = :token
+                 WHERE
+                       resourcekey = :resourcekey AND
+                       (owner IS NULL OR expires < :now)';
+
+        do {
+            $now = time();
+            $params['now'] = $now;
+            $this->db->execute($sql, $params);
+
+            $countparams = array('owner' => $token, 'resourcekey' => $resource);
+            $result = $this->db->count_records('lock_db', $countparams);
+            $locked = $result === 1;
+            if (!$locked) {
+                usleep(rand(10000, 250000)); // Sleep between 10 and 250 milliseconds.
+            }
+            // Try until the giveup time.
+        } while (!$locked && $now < $giveuptime);
+
+        if ($locked) {
+            $this->openlocks[$token] = 1;
+            return new lock($token, $this);
+        }
+
+        return false;
+    }
+
+    /**
+     * Release a lock that was previously obtained with @lock.
+     * @param lock $lock - a lock obtained from this factory.
+     * @return boolean - true if the lock is no longer held (including if it was never held).
+     */
+    public function release_lock(lock $lock) {
+        $params = array('noexpires' => null,
+                        'token' => $lock->get_key(),
+                        'noowner' => null);
+
+        $sql = 'UPDATE {lock_db}
+                    SET
+                        expires = :noexpires,
+                        owner = :noowner
+                    WHERE
+                        owner = :token';
+        $result = $this->db->execute($sql, $params);
+        if ($result) {
+            unset($this->openlocks[$lock->get_key()]);
+        }
+        return $result;
+    }
+
+    /**
+     * Extend a lock that was previously obtained with @lock.
+     * @param lock $lock - a lock obtained from this factory.
+     * @param int $maxlifetime - the new lifetime for the lock (in seconds).
+     * @return boolean - true if the lock was extended.
+     */
+    public function extend_lock(lock $lock, $maxlifetime = 86400) {
+        $now = time();
+        $expires = $now + $maxlifetime;
+        $params = array('expires' => $expires,
+                        'token' => $lock->get_key());
+
+        $sql = 'UPDATE {lock_db}
+                    SET
+                        expires = :expires,
+                    WHERE
+                        owner = :token';
+
+        $this->db->execute($sql, $params);
+        $countparams = array('owner' => $lock->get_key());
+        $result = $this->count_records('lock_db', $countparams);
+
+        return $result === 0;
+    }
+
+    /**
+     * Auto release any open locks on shutdown.
+     * This is required, because we may be using persistent DB connections.
+     */
+    public function auto_release() {
+        // Called from the shutdown handler. Must release all open locks.
+        foreach ($this->openlocks as $key => $unused) {
+            $lock = new lock($key, $this);
+            $this->release_lock($lock);
+        }
+    }
+}

lib/classes/lock/file_lock_factory.php

+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Flock based file locking factory.
+ *
+ * The file lock factory returns file locks locked with the flock function. Works OK, except on some
+ * NFS, exotic shared storage and exotic server OSes (like windows). On windows, a second attempt to get a
+ * lock will block indefinitely instead of timing out.
+ *
+ * @package    core
+ * @category   lock
+ * @copyright  Damyon Wiese 2013
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core\lock;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Flock based file locking factory.
+ *
+ * The file lock factory returns file locks locked with the flock function. Works OK, except on some
+ * NFS, exotic shared storage and exotic server OSes (like windows). On windows, a second attempt to get a
+ * lock will block indefinitely instead of timing out.
+ *
+ * @package   core
+ * @category  lock
+ * @copyright Damyon Wiese 2013
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class file_lock_factory implements lock_factory {
+
+    /** @var string $type - The type of lock, e.g. cache, cron, session. */
+    protected $type;
+
+    /** @var string $lockdirectory - Full system path to the directory used to store file locks. */
+    protected $lockdirectory;
+
+    /** @var boolean $verbose - If true, debugging info about the owner of the lock will be written to the lock file. */
+    protected $verbose;
+
+    /**
+     * Create this lock factory.
+     *
+     * @param string $type - The type, e.g. cron, cache, session
+     */
+    public function __construct($type) {
+        global $CFG;
+
+        $this->type = $type;
+        if (!isset($CFG->file_lock_root)) {
+            $this->lockdirectory = $CFG->dataroot . '/lock';
+        } else {
+            $this->lockdirectory = $CFG->file_lock_root;
+        }
+        $this->verbose = false;
+        if ($CFG->debugdeveloper) {
+            $this->verbose = true;
+        }
+    }
+
+    /**
+     * Return information about the blocking behaviour of the lock type on this platform.
+     * @return boolean - False if attempting to get a lock will block indefinitely.
+     */
+    public function supports_timeout() {
+        global $CFG;
+
+        return $CFG->ostype !== 'WINDOWS';
+    }
+
+    /**
+     * This lock type will be automatically released when a process ends.
+     * @return boolean - True
+     */
+    public function supports_auto_release() {
+        return true;
+    }
+
+    /**
+     * Is available.
+     * @return boolean - True if this lock type is available in this environment.
+     */
+    public function is_available() {
+        return true;
+    }
+
+    /**
+     * Multiple locks for the same resource cannot be held from a single process.
+     * @return boolean - False
+     */
+    public function supports_recursion() {
+        return false;
+    }
+
+    /**
+     * Get some info that might be useful for debugging.
+     * @return boolean - string
+     */
+    protected function get_debug_info() {
+        return 'host:' . php_uname('n') . ', pid:' . getmypid() . ', time:' . time();
+    }
+
+    /**
+     * Get a lock within the specified timeout or return false.
+     * @param string $resource - The identifier for the lock. Should use frankenstyle prefix.
+     * @param int $timeout - The number of seconds to wait for a lock before giving up.
+     * @param int $maxlifetime - Unused by this lock type.
+     * @return boolean - true if a lock was obtained.
+     */
+    public function get_lock($resource, $timeout, $maxlifetime = 86400) {
+        global $CFG;
+
+        $giveuptime = time() + $timeout;
+
+        $hash = md5($this->type . '_' . $resource);
+        $lockdir = $this->lockdirectory . '/' . substr($hash, 0, 2);
+
+        if (!check_dir_exists($lockdir, true, true)) {
+            return false;
+        }
+
+        $lockfilename = $lockdir . '/' . $hash;
+
+        $filehandle = fopen($lockfilename, "wb");
+
+        // Could not open the lock file.
+        if (!$filehandle) {
+            return false;
+        }
+
+        do {
+            // Will block on windows. So sad.
+            $wouldblock = false;
+            $locked = flock($filehandle, LOCK_EX | LOCK_NB, $wouldblock);
+            if (!$locked && $wouldblock) {
+                usleep(rand(10000, 250000)); // Sleep between 10 and 250 milliseconds.
+            }
+            // Try until the giveup time.
+        } while (!$locked && $wouldblock && time() < $giveuptime);
+
+        if (!$locked) {
+            fclose($filehandle);
+            return false;
+        }
+        if ($this->verbose) {
+            fwrite($filehandle, $this->get_debug_info());
+        }
+        return new lock($filehandle, $this);
+    }
+
+    /**
+     * Release a lock that was previously obtained with @lock.
+     * @param lock $lock - A lock obtained from this factory.
+     * @return boolean - true if the lock is no longer held (including if it was never held).
+     */
+    public function release_lock(lock $lock) {
+        $handle = $lock->get_key();
+
+        if (!$handle) {
+            // We didn't have a lock.
+            return false;
+        }
+
+        $result = flock($handle, LOCK_UN);
+        fclose($handle);
+        return $result;
+    }
+
+    /**
+     * Extend a lock that was previously obtained with @lock.
+     * @param lock $lock - not used
+     * @param int $maxlifetime - not used
+     * @return boolean - true if the lock was extended.
+     */
+    public function extend_lock(lock $lock, $maxlifetime = 86400) {
+        // Not supported by this factory.
+        return false;
+    }
+
+}

lib/classes/lock/lock.php

+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Class representing a lock
+ *
+ * The methods available for a specific lock type are only known by it's factory.
+ *
+ * @package    core
+ * @copyright  Damyon Wiese 2013
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+
+namespace core\lock;
+
+defined('MOODLE_INTERNAL') || die();
+
+/**
+ * Class representing a lock
+ *
+ * The methods available for a specific lock type are only known by it's factory.
+ *
+ * @package   core
+ * @category  lock
+ * @copyright Damyon Wiese 2013
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class lock {
+
+    /** @var string|int $key A uniq key representing a held lock */
+    protected $key = '';
+
+    /** @var lock_factory $factory The factory that generated this lock */
+    protected $factory;
+
+    /** @var bool $released Has this lock been released? If a lock falls out of scope without being released - show a warning. */
+    protected $released;
+
+    /**
+     * Construct a lock containing the unique key required to release it.
+     * @param string $key - The lock key.
+     * @param lock_factory $factory - The factory that generated this lock.
+     */
+    public function __construct($key, $factory) {
+        $this->factory = $factory;
+        $this->key = $key;
+        $this->released = false;
+    }
+
+    /**
+     * Return the unique key representing this lock.
+     * @return string|int lock key.
+     */
+    public function get_key() {
+        return $this->key;
+    }
+
+    /**
+     * Extend the lifetime of this lock. Not supported by all factories.
+     * @param int $maxlifetime - the new lifetime for the lock (in seconds).
+     * @return bool
+     */
+    public function extend($maxlifetime = 86400) {
+        if ($this->factory) {
+            return $this->factory->extend_lock($this, $maxlifetime);
+        }
+        return false;
+    }
+
+    /**
+     * Release this lock
+     * @return bool
+     */
+    public function release() {
+        $this->released = true;
+        if (empty($this->factory)) {
+            return false;
+        }
+        $result = $this->factory->release_lock($this);
+        // Release any held references to the factory.
+        unset($this->factory);
+        $this->factory = null;
+        $this->key = '';
+        return $result;
+    }
+
+    /**
+     * Print debugging if this lock falls out of scope before being released.
+     */
+    public function __destruct() {
+        if (!$this->released && defined('PHPUNIT_TEST')) {
+            $this->release();
+            throw new \coding_exception('\core\lock\lock(' . $this->key . ') has fallen out of scope ' .
+                                        'without being released.' . "\n" .
+                                        'Locks must ALWAYS be released by calling $mylock->release().');
+        }
+    }
+
+}

lib/classes/lock/lock_config.php

+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+/**
+ * Lock configuration class, used to get an instance of the currently configured lock factory.
+ *
+ * @package    core
+ * @category   lock
+ * @copyright  Damyon Wiese 2013
+ * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */