Commit c1109657 authored by Eloy Lafuente (stronk7)'s avatar Eloy Lafuente (stronk7)
Browse files

Merge branch 'MDL-72619-master' of https://github.com/sammarshallou/moodle

parents faddd24a 1b94bb8c
......@@ -726,6 +726,8 @@ if ($hassiteconfig) {
$ADMIN->add('modules', new admin_category('cache', new lang_string('caching', 'cache')));
$ADMIN->add('cache', new admin_externalpage('cacheconfig', new lang_string('cacheconfig', 'cache'), $CFG->wwwroot .'/cache/admin.php'));
$ADMIN->add('cache', new admin_externalpage('cachetestperformance', new lang_string('testperformance', 'cache'), $CFG->wwwroot . '/cache/testperformance.php'));
$ADMIN->add('cache', new admin_externalpage('cacheusage',
new lang_string('cacheusage', 'cache'), $CFG->wwwroot . '/cache/usage.php'));
$ADMIN->add('cache', new admin_category('cachestores', new lang_string('cachestores', 'cache')));
$ADMIN->locate('cachestores')->set_sorting(true);
foreach (core_component::get_plugin_list('cachestore') as $plugin => $path) {
......
......@@ -65,7 +65,7 @@ $title = array_key_exists('title', $forminfo) ? $forminfo['title'] : new lang_st
$PAGE->set_title($title);
$PAGE->set_heading($SITE->fullname);
/* @var core_cache_renderer $renderer */
/** @var \core_cache\output\renderer $renderer */
$renderer = $PAGE->get_renderer('core_cache');
echo $renderer->header();
......
......@@ -400,8 +400,32 @@ abstract class administration_helper extends cache_helper {
/**
* This function must be implemented to display the cache admin page.
*
* @param core_cache_renderer $renderer the renderer used to generate the page.
* @param \core_cache\output\renderer $renderer the renderer used to generate the page.
* @return string the HTML for the page.
*/
public abstract function generate_admin_page(\core_cache_renderer $renderer): string;
abstract public function generate_admin_page(\core_cache\output\renderer $renderer): string;
/**
* Gets usage information about the whole cache system.
*
* This is a slow function and should only be used on an admin information page.
*
* The returned array lists all cache definitions with fields 'cacheid' and 'stores'. For
* each store, the following fields are available:
*
* - name (store name)
* - class (e.g. cachestore_redis)
* - supported (true if we have any information)
* - items (number of items stored)
* - mean (mean size of item)
* - sd (standard deviation for item sizes)
* - margin (margin of error for mean at 95% confidence)
* - storetotal (total usage for store if known, otherwise null)
*
* The storetotal field will be the same for every cache that uses the same store.
*
* @param int $samplekeys Number of keys to sample when checking size of large caches
* @return array Details of cache usage
*/
abstract public function get_usage(int $samplekeys): array;
}
......@@ -31,7 +31,7 @@
namespace core_cache\local;
defined('MOODLE_INTERNAL') || die();
use cache_store, cache_factory, cache_config_writer, cache_helper, core_cache_renderer;
use cache_store, cache_factory, cache_config_writer, cache_helper;
/**
* A cache helper for administration tasks
......@@ -765,10 +765,10 @@ class administration_display_helper extends \core_cache\administration_helper {
/**
* Outputs the main admin page by generating it through the renderer.
*
* @param core_cache_renderer $renderer the renderer to use to generate the page.
* @param \core_cache\output\renderer $renderer the renderer to use to generate the page.
* @return string the HTML for the admin page.
*/
public function generate_admin_page(core_cache_renderer $renderer): string {
public function generate_admin_page(\core_cache\output\renderer $renderer): string {
$context = \context_system::instance();
$html = '';
......@@ -792,4 +792,87 @@ class administration_display_helper extends \core_cache\administration_helper {
return $html;
}
}
\ No newline at end of file
/**
* Gets usage information about the whole cache system.
*
* This is a slow function and should only be used on an admin information page.
*
* The returned array lists all cache definitions with fields 'cacheid' and 'stores'. For
* each store, the following fields are available:
*
* - name (store name)
* - class (e.g. cachestore_redis)
* - supported (true if we have any information)
* - items (number of items stored)
* - mean (mean size of item)
* - sd (standard deviation for item sizes)
* - margin (margin of error for mean at 95% confidence)
* - storetotal (total usage for store if known, otherwise null)
*
* The storetotal field will be the same for every cache that uses the same store.
*
* @param int $samplekeys Number of keys to sample when checking size of large caches
* @return array Details of cache usage
*/
public function get_usage(int $samplekeys): array {
$results = [];
$factory = cache_factory::instance();
// Check the caches we already have an instance of, so we don't make another one...
$got = $factory->get_caches_in_use();
$gotid = [];
foreach ($got as $longid => $unused) {
// The IDs here can be of the form cacheid/morestuff if there are parameters in the
// cache. Any entry for a cacheid is good enough to consider that we don't need to make
// another entry ourselves, so we remove the extra bits and track the basic cache id.
$gotid[preg_replace('~^([^/]+/[^/]+)/.*$~', '$1', $longid)] = true;
}
$storetotals = [];
$config = $factory->create_config_instance();
foreach ($config->get_definitions() as $configdetails) {
if (!array_key_exists($configdetails['component'] . '/' . $configdetails['area'], $gotid)) {
// Where possible (if it doesn't need identifiers), make an instance of the cache, otherwise
// we can't get the store instances for it (and it won't show up in the list).
if (empty($configdetails['requireidentifiers'])) {
\cache::make($configdetails['component'], $configdetails['area']);
}
}
$definition = $factory->create_definition($configdetails['component'], $configdetails['area']);
$stores = $factory->get_store_instances_in_use($definition);
// Create object for results about this cache definition.
$currentresult = (object)['cacheid' => $definition->get_id(), 'stores' => []];
$results[$currentresult->cacheid] = $currentresult;
/** @var cache_store $store */
foreach ($stores as $store) {
// Skip static cache.
if ($store instanceof \cachestore_static) {
continue;
}
// Get cache size details from store.
$currentstore = $store->cache_size_details($samplekeys);
// Add in basic information about store.
$currentstore->name = $store->my_name();
$currentstore->class = get_class($store);
// Add in store total.
if (!array_key_exists($currentstore->name, $storetotals)) {
$storetotals[$currentstore->name] = $store->store_total_size();
}
$currentstore->storetotal = $storetotals[$currentstore->name];
$currentresult->stores[] = $currentstore;
}
}
ksort($results);
return $results;
}
}
......@@ -14,28 +14,28 @@
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* The Cache renderer.
*
* This file is part of Moodle's cache API, affectionately called MUC.
*
* @package core
* @category cache
* @copyright 2012 Sam Hemelryk
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
defined('MOODLE_INTERNAL') || die();
namespace core_cache\output;
use cache_factory;
use cache_store;
use context;
use core_collator;
use html_table;
use html_table_cell;
use html_table_row;
use html_writer;
use lang_string;
use moodle_url;
use single_select;
/**
* The cache renderer (mainly admin interfaces).
*
* @package core
* @category cache
* @package core_cache
* @copyright 2012 Sam Hemelryk
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class core_cache_renderer extends plugin_renderer_base {
class renderer extends \plugin_renderer_base {
/**
* Displays store summaries.
......@@ -413,4 +413,138 @@ class core_cache_renderer extends plugin_renderer_base {
$html .= html_writer::end_div();
return $html;
}
/**
* Creates the two tables which display on the usage page.
*
* @param array $usage Usage information (from cache_helper::usage)
* @return array Array of 2 tables (main and summary table)
* @throws \coding_exception
*/
public function usage_tables(array $usage): array {
$table = new \html_table();
$table->id = 'usage_main';
$table->head = [
get_string('definition', 'cache'),
get_string('storename', 'cache'),
get_string('plugin', 'cache'),
get_string('usage_items', 'cache'),
get_string('usage_mean', 'cache'),
get_string('usage_sd', 'cache'),
get_string('usage_total', 'cache'),
get_string('usage_totalmargin', 'cache')];
$table->align = [
'left', 'left', 'left',
'right', 'right', 'right', 'right', 'right'
];
$table->data = [];
$summarytable = new \html_table();
$summarytable->id = 'usage_summary';
$summarytable->head = [
get_string('storename', 'cache'),
get_string('plugin', 'cache'),
get_string('usage_total', 'cache'),
get_string('usage_realtotal', 'cache')
];
$summarytable->align = [
'left', 'left',
'right', 'right',
];
$summarytable->data = [];
$summarytable->attributes['class'] = 'generaltable w-auto';
$storetotals = [];
// We will highlight all cells that are more than 2% of total size, so work that out first.
$total = 0;
foreach ($usage as $definition) {
foreach ($definition->stores as $storedata) {
$total += $storedata->items * $storedata->mean;
}
}
$highlightover = round($total / 50);
foreach ($usage as $definition) {
foreach ($definition->stores as $storedata) {
$row = [];
$row[] = s($definition->cacheid);
$row[] = s($storedata->name);
$row[] = s($storedata->class);
if (!$storedata->supported) {
// We don't have data for this store because it isn't searchable.
$row[] = '-';
} else {
$row[] = $storedata->items;
}
if ($storedata->items) {
$row[] = display_size(round($storedata->mean));
if ($storedata->items > 1) {
$row[] = display_size(round($storedata->sd));
} else {
$row[] = '';
}
$cellsize = round($storedata->items * $storedata->mean);
$row[] = display_size($cellsize, 1, 'MB');
if (!array_key_exists($storedata->name, $storetotals)) {
$storetotals[$storedata->name] = (object)[
'plugin' => $storedata->class,
'total' => 0,
'storetotal' => $storedata->storetotal,
];
}
$storetotals[$storedata->name]->total += $cellsize;
} else {
$row[] = '';
$row[] = '';
$cellsize = 0;
$row[] = '';
}
if ($storedata->margin) {
// Plus or minus.
$row[] = '&#xb1;' . display_size($storedata->margin * $storedata->items, 1, 'MB');
} else {
$row[] = '';
}
$htmlrow = new \html_table_row($row);
if ($cellsize > $highlightover) {
$htmlrow->attributes = ['class' => 'table-warning'];
}
$table->data[] = $htmlrow;
}
}
ksort($storetotals);
foreach ($storetotals as $storename => $storedetails) {
$row = [s($storename), s($storedetails->plugin)];
$row[] = display_size($storedetails->total, 1, 'MB');
if ($storedetails->storetotal !== null) {
$row[] = display_size($storedetails->storetotal, 1, 'MB');
} else {
$row[] = '-';
}
$summarytable->data[] = $row;
}
return [$table, $summarytable];
}
/**
* Renders the usage page.
*
* @param \html_table $maintable Main table
* @param \html_table $summarytable Summary table
* @param \moodleform $samplesform Form to select number of samples
* @return string HTML for page
*/
public function usage_page(\html_table $maintable, \html_table $summarytable, \moodleform $samplesform): string {
$data = [
'maintable' => \html_writer::table($maintable),
'summarytable' => \html_writer::table($summarytable),
'samplesform' => $samplesform->render()
];
return $this->render_from_template('core_cache/usage', $data);
}
}
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Form for usage page to select number of samples.
*
* @package core_cache
* @copyright 2021 The Open University
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core_cache\output;
defined('MOODLE_INTERNAL') || die();
require_once($CFG->libdir . '/formslib.php');
/**
* Form for usage page to select number of samples.
*
* @package core_cache
*/
class usage_samples_form extends \moodleform {
/**
* Constructor sets form up to use GET request to current page.
*/
public function __construct() {
parent::__construct(null, null, 'get');
}
/**
* Adds controls to form.
*/
protected function definition() {
$mform = $this->_form;
$radioarray = [];
foreach ([50, 100, 200, 500, 1000] as $samples) {
$radioarray[] = $mform->createElement('radio', 'samples', '', $samples, $samples);
}
$mform->setDefault('samples', 50);
$mform->addGroup($radioarray, 'samplesradios', get_string('usage_samples', 'cache'), [' '], false);
$mform->addElement('submit', 'submit', get_string('update'));
}
}
......@@ -386,6 +386,123 @@ abstract class cache_store implements cache_store_interface {
return array();
}
/**
* Estimates the storage size used within this cache if the given value is stored with the
* given key.
*
* This function is not exactly accurate; it does not necessarily take into account all the
* overheads involved. It is only intended to give a good idea of the relative size of
* different caches.
*
* The default implementation serializes both key and value and sums the lengths (as a rough
* estimate which is probably good enough for everything unless the cache offers compression).
*
* @param mixed $key Key
* @param mixed $value Value
* @return int Size in bytes
*/
public function estimate_stored_size($key, $value): int {
return strlen(serialize($key)) + strlen(serialize($value));
}
/**
* Gets the amount of memory/storage currently used by this cache store if known.
*
* This value should be obtained quickly from the store itself, if available.
*
* This is the total memory usage of the entire store, not for ther specific cache in question.
*
* Where not supported (default), will always return null.
*
* @return int|null Amount of memory used in bytes or null
*/
public function store_total_size(): ?int {
return null;
}
/**
* Gets the amount of memory used by this specific cache within the store, if known.
*
* This function may be slow and should not be called in normal usage, only for administration
* pages. The value is usually an estimate, and may not be available at all.
*
* When estimating, a number of sample items will be used for the estimate. If set to 50
* (default), then this function will retrieve 50 random items and use that to estimate the
* total size.
*
* The return value has the following fields:
* - supported (true if any other values are completed)
* - items (number of items)
* - mean (mean size of one item in bytes)
* - sd (standard deviation of item size in bytes, based on sample)
* - margin (95% confidence margin for mean - will be 0 if exactly computed)
*
* @param int $samplekeys Number of samples to use
* @return stdClass Object with information about the store size
*/
public function cache_size_details(int $samplekeys = 50): stdClass {
$result = (object)[
'supported' => false,
'items' => 0,
'mean' => 0,
'sd' => 0,
'margin' => 0
];
// If this cache isn't searchable, we don't know the answer.
if (!$this->is_searchable()) {
return $result;
}
$result->supported = true;
// Get all the keys for the cache.
$keys = $this->find_all();
$result->items = count($keys);
// Don't do anything else if there are no items.
if ($result->items === 0) {
return $result;
}
// Select N random keys.
$exact = false;
if ($result->items <= $samplekeys) {
$samples = $keys;
$exact = true;
} else {
$indexes = array_rand($keys, $samplekeys);
$samples = [];
foreach ($indexes as $index) {
$samples[] = $keys[$index];
}
}
// Get the random items from cache and estimate the size of each.
$sizes = [];
foreach ($samples as $samplekey) {
$value = $this->get($samplekey);
$sizes[] = $this->estimate_stored_size($samplekey, $value);
}
$number = count($sizes);
// Calculate the mean and standard deviation.
$result->mean = array_sum($sizes) / $number;
$squarediff = 0;
foreach ($sizes as $size) {
$squarediff += ($size - $result->mean) ** 2;
}
$squarediff /= $number;
$result->sd = sqrt($squarediff);
// If it's not exact, also calculate the confidence interval.
if (!$exact) {
// 95% confidence has a Z value of 1.96.
$result->margin = (1.96 * $result->sd) / sqrt($number);
}
return $result;
}
/**
* Returns true if this cache store instance is both suitable for testing, and ready for testing.
*
......
......@@ -811,4 +811,57 @@ class cachestore_file extends cache_store implements cache_is_key_aware, cache_i
}
return $return;
}
/**
* Gets total size for the directory used by the cache store.
*
* @return int Total size in bytes
*/
public function store_total_size(): ?int {
return get_directory_size($this->filestorepath);
}
/**
* Gets total size for a specific cache.
*
* With the file cache we can just look at the directory listing without having to
* actually load any files, so the $samplekeys parameter is ignored.
*
* @param int $samplekeys Unused
* @return stdClass Cache details
*/
public function cache_size_details(int $samplekeys = 50): stdClass {
$result = (object)[
'supported' => true,
'items' => 0,
'mean' => 0,
'sd' => 0,
'margin' => 0
];
// Find all the files in this cache.
$this->ensure_path_exists();
$files = glob($this->glob_keys_pattern(), GLOB_MARK | GLOB_NOSORT);
if ($files === false || count($files) === 0) {
return $result;
}
// Get the sizes and count of files.
$sizes = [];
foreach ($files as $file) {
$result->items++;
$sizes[] = filesize($file);
}
// Work out mean and standard deviation.
$total = array_sum($sizes);
$result->mean = $total / $result->items;
$squarediff = 0;
foreach ($sizes as $size) {
$squarediff += ($size - $result->mean) ** 2;
}
$squarediff /= $result->items;
$result->sd = sqrt($squarediff);
return $result;
}
}
......@@ -606,7 +606,7 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
$count = 0;
$batches = 0;
$timebefore = microtime(true);
$memorybefore = $this->get_used_memory();
$memorybefore = $this->store_total_size();
do {
$keys = $this->redis->zRangeByScore($this->hash . self::TTL_SUFFIX, 0, $limit,
['limit' => [0, self::TTL_EXPIRE_BATCH]]);
......@@ -614,7 +614,7 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
$count += count($keys);
$batches++;
} while (count($keys) === self::TTL_EXPIRE_BATCH);
$memoryafter = $this->get_used_memory();
$memoryafter = $this->store_total_size();
$timeafter = microtime(true);
$result = ['keys' => $count, 'batches' => $batches, 'time' => $timeafter - $timebefore];
......@@ -657,12 +657,29 @@ class cachestore_redis extends cache_store implements cache_is_key_aware, cache_
}
}
/**
* Estimates the stored size, taking into account whether compression is turned on.
*