Commit e90a0a74 authored by Adrian Greeve's avatar Adrian Greeve
Browse files

MDL-62491 core_privacy: Add html files to the user data export.

This adds html to the data export that allows for easier navigation
and reading of data.
parent 129783b9
......@@ -22,6 +22,10 @@
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
$string['broughtbymoodle'] = 'This data export provided by Moodle';
$string['exportfrom'] = 'Exported from {$a}';
$string['exporttime'] = 'Exported on {$a}';
$string['exportuser'] = 'Data for {$a}';
$string['privacy:metadata'] = 'The privacy subsystem does not store any data of its own and is designed to act as a channel between components and the interface used to describe, export, and remove their data.';
$string['trace:done'] = 'Complete';
$string['trace:exportcomplete'] = 'Export complete';
......@@ -32,4 +36,6 @@ $string['trace:processingcomponent'] = 'Processing {$a->component} ({$a->progres
$string['trace:fetchcomponents'] = 'Fetching {$a->total} components ({$a->datetime})';
$string['trace:deletingapproved'] = 'Performing removal of approved {$a->total} contexts ({$a->datetime})';
$string['trace:deletingcontext'] = 'Performing removal of context from {$a->total} components ({$a->datetime})';
$string['navigation'] = 'Navigation';
$string['privacy:subsystem:empty'] = 'This subsystem does not store any data.';
$string['viewdata'] = 'Click on a link in the navigation to view data.';
......@@ -13,4 +13,6 @@ Description of import of various jQuery libraries into Moodle:
5/ open http://127.0.0.1/lib/tests/other/jquerypage.php
6/ Update the version of jquery in core_privacy\local\request\moodle_content_writer::write_html_data()
Petr Skoda
......@@ -2,3 +2,4 @@ Description of import into Moodle:
// Download from https://requirejs.org/docs/download.html
// Put the require.js and require.min.js and LICENSE file in this folder.
// Check if MDL-60458 workaround can be removed.
// Check that core_privacy\local\request\moodle_content_writer::write_html_data() does not need to be updated.
......@@ -52,6 +52,11 @@ class moodle_content_writer implements content_writer {
*/
protected $files = [];
/**
* @var array The list of plugins that have been checked to see if they are installed.
*/
protected $checkedplugins = [];
/**
* Constructor for the content writer.
*
......@@ -162,7 +167,17 @@ class moodle_content_writer implements content_writer {
* @return string The processed string
*/
public function rewrite_pluginfile_urls(array $subcontext, $component, $filearea, $itemid, $text) : string {
return str_replace('@@PLUGINFILE@@/', $this->get_files_target_url($component, $filearea, $itemid).'/', $text);
// Need to take into consideration the subcontext to provide the full path to this file.
$subcontextpath = '';
if (!empty($subcontext)) {
$subcontextpath = DIRECTORY_SEPARATOR . implode(DIRECTORY_SEPARATOR, $subcontext);
}
$path = $this->get_context_path();
$path = implode(DIRECTORY_SEPARATOR, $path) . $subcontextpath;
$returnstring = $path . DIRECTORY_SEPARATOR . $this->get_files_target_url($component, $filearea, $itemid) . '/';
$returnstring = clean_param($returnstring, PARAM_PATH);
return str_replace('@@PLUGINFILE@@/', $returnstring, $text);
}
/**
......@@ -245,7 +260,7 @@ class moodle_content_writer implements content_writer {
$contexts = array_reverse($this->context->get_parent_contexts(true));
foreach ($contexts as $context) {
$name = $context->get_context_name();
$id = $context->id;
$id = '_.' . $context->id;
$path[] = shorten_filename(clean_param("{$name} {$id}", PARAM_FILE), MAX_FILENAME_SIZE, true);
}
......@@ -263,6 +278,24 @@ class moodle_content_writer implements content_writer {
$subcontext = shorten_filenames($subcontext, MAX_FILENAME_SIZE, true);
$name = shorten_filename($name, MAX_FILENAME_SIZE, true);
// This weird code is to look for a subcontext that contains a number and append an '_' to the front.
// This is because there seems to be some weird problem with array_merge_recursive used in finalise_content().
$subcontext = array_map(function($data) {
if (stripos($data, DIRECTORY_SEPARATOR) !== false) {
$newpath = explode(DIRECTORY_SEPARATOR, $data);
$newpath = array_map(function($value) {
if (is_numeric($value)) {
return '_' . $value;
}
return $value;
}, $newpath);
return implode(DIRECTORY_SEPARATOR, $newpath);
} else if (is_numeric($data)) {
$data = '_' . $data;
}
return $data;
}, $subcontext);
// Combine the context path, and the subcontext data.
$path = array_merge(
$this->get_context_path(),
......@@ -331,7 +364,7 @@ class moodle_content_writer implements content_writer {
$parts = ['_files', $filearea];
if (!empty($itemid)) {
$parts[] = $itemid;
$parts[] = '_' . $itemid;
}
return implode('/', $parts);
......@@ -350,12 +383,308 @@ class moodle_content_writer implements content_writer {
$this->files[$path] = $targetpath;
}
/**
* Copy a file to the specified path.
*
* @param array $path Current location of the file.
* @param array $destination Destination path to copy the file to.
*/
protected function copy_data(array $path, array $destination) {
// Do we not have a moodle function to do something like this?
$systempath = getcwd();
// This is likely to be running from admin/cli.
if (stripos($systempath, 'admin' . DIRECTORY_SEPARATOR . 'cli') !== false) {
$bits = explode('admin' . DIRECTORY_SEPARATOR . 'cli', $systempath);
$systempath = implode('', $bits);
}
$filename = array_pop($destination);
$destdirectory = implode(DIRECTORY_SEPARATOR, $destination);
$fulldestination = $this->path . DIRECTORY_SEPARATOR . $destdirectory;
check_dir_exists($fulldestination, true, true);
$fulldestination .= $filename;
$currentpath = $systempath . DIRECTORY_SEPARATOR . implode(DIRECTORY_SEPARATOR, $path);
copy($currentpath, $fulldestination);
$this->files[$destdirectory . DIRECTORY_SEPARATOR . $filename] = $fulldestination;
}
/**
* This creates three different bits of data from all of the files that will be
* exported.
* $tree - A multidimensional array of the navigation tree structure.
* $treekey - An array with the short path of the file and element data for
* html (data_file_{number} or 'No var')
* $allfiles - All *.json files that need to be added as an index to be referenced
* by the js files to display the user data.
*
* @return array returns a tree, tree key, and a list of all files.
*/
protected function prepare_for_export() : Array {
$tree = [];
$treekey = [];
$allfiles = [];
$i = 1;
foreach ($this->files as $shortpath => $fullfile) {
// Generate directory tree as an associative array.
$items = explode(DIRECTORY_SEPARATOR, $shortpath);
$newitems = $this->condense_array($items);
$tree = array_merge_recursive($tree, $newitems);
if (is_string($fullfile)) {
$filearray = explode(DIRECTORY_SEPARATOR, $shortpath);
$filename = array_pop($filearray);
$filenamearray = explode('.', $filename);
// Don't process files that are not json files.
if (end($filenamearray) !== 'json') {
continue;
}
// Chop the last two characters of the extension. json => js.
$filename = substr($filename, 0, -2);
array_push($filearray, $filename);
$newshortpath = implode(DIRECTORY_SEPARATOR, $filearray);
$varname = 'data_file_' . $i;
$i++;
$quicktemp = clean_param($shortpath, PARAM_PATH);
$treekey[$quicktemp] = $varname;
$allfiles[$varname] = clean_param($newshortpath, PARAM_PATH);
// Need to load up the current json file and add a variable (varname mentioned above) at the start.
// Then save it as a js file.
$content = $this->get_file_content($fullfile);
$jsondecodedcontent = json_decode($content);
$jsonencodedcontent = json_encode($jsondecodedcontent, JSON_PRETTY_PRINT);
$variablecontent = 'var ' . $varname . ' = ' . $jsonencodedcontent;
$this->write_data($newshortpath, $variablecontent);
} else {
$treekey[$shortpath] = 'No var';
}
}
return [$tree, $treekey, $allfiles];
}
/**
* Add more detail to the tree to help with sorting and display in the renderer.
*
* @param array $tree The file structure currently as a multidimensional array.
* @param array $treekey An array of the current file paths.
* @param array $currentkey The current short path of the tree.
* @return array An array of objects that has additional data.
*/
protected function make_tree_object(array $tree, array $treekey, array $currentkey = []) : Array {
$newtree = [];
// Try to extract the context id and then add the context object.
$addcontext = function($index, $object) {
if (stripos($index, '_.') !== false) {
$namearray = explode('_.', $index);
$contextid = array_pop($namearray);
if (is_numeric($contextid)) {
$object[$index]->name = implode('_.', $namearray);
$object[$index]->context = \context::instance_by_id($contextid);
}
} else {
$object[$index]->name = $index;
}
};
// Just add the final data to the tree object.
$addfinalfile = function($directory, $treeleaf, $file) use ($treekey) {
$url = implode(DIRECTORY_SEPARATOR, $directory);
$url = clean_param($url, PARAM_PATH);
$treeleaf->name = $file;
$treeleaf->itemtype = 'item';
$gokey = $url . DIRECTORY_SEPARATOR . $file;
if (isset($treekey[$gokey]) && $treekey[$gokey] !== 'No var') {
$treeleaf->datavar = $treekey[$gokey];
} else {
$treeleaf->url = new \moodle_url($url . DIRECTORY_SEPARATOR . $file);
}
};
foreach ($tree as $key => $value) {
$newtree[$key] = new \stdClass();
if (is_array($value)) {
$newtree[$key]->itemtype = 'treeitem';
// The array merge recursive adds a numeric index, and so we only add to the current
// key if it is now numeric.
$currentkey = is_numeric($key) ? $currentkey : array_merge($currentkey, [$key]);
// Try to extract the context id and then add the context object.
$addcontext($key, $newtree);
$newtree[$key]->children = $this->make_tree_object($value, $treekey, $currentkey);
if (!is_numeric($key)) {
// We're heading back down the tree, so remove the last key.
array_pop($currentkey);
}
} else {
// If the key is not numeric then we want to add a directory and put the file under that.
if (!is_numeric($key)) {
$newtree[$key]->itemtype = 'treeitem';
// Try to extract the context id and then add the context object.
$addcontext($key, $newtree);
array_push($currentkey, $key);
$newtree[$key]->children[$value] = new \stdClass();
$addfinalfile($currentkey, $newtree[$key]->children[$value], $value);
array_pop($currentkey);
} else {
// If the key is just a number then we just want to show the file instead.
$addfinalfile($currentkey, $newtree[$key], $value);
}
}
}
return $newtree;
}
/**
* Sorts the tree list into an order that makes more sense.
* Order is:
* 1 - Items with a context first, the lower the number the higher up the tree.
* 2 - Items that are directories.
* 3 - Items that are log directories.
* 4 - Links to a page.
*
* @param array $tree The tree structure to order.
*/
protected function sort_my_list(array &$tree) {
uasort($tree, function($a, $b) {
if (isset($a->context) && isset($b->context)) {
return $a->context->contextlevel <=> $b->context->contextlevel;
}
if (isset($a->context) && !isset($b->context)) {
return -1;
}
if (isset($b->context) && !isset($a->context)) {
return 1;
}
if ($a->itemtype == 'treeitem' && $b->itemtype == 'treeitem') {
// Ugh need to check that this plugin has not been uninstalled.
if ($this->check_plugin_is_installed('tool_log')) {
if (trim($a->name) == get_string('privacy:path:logs', 'tool_log')) {
return 1;
} else if (trim($b->name) == get_string('privacy:path:logs', 'tool_log')) {
return -1;
}
return 0;
}
}
if ($a->itemtype == 'treeitem' && $b->itemtype == 'item') {
return -1;
}
if ($b->itemtype == 'treeitem' && $a->itemtype == 'item') {
return 1;
}
return 0;
});
foreach ($tree as $treeobject) {
if (isset($treeobject->children)) {
$this->sort_my_list($treeobject->children);
}
}
}
/**
* Check to see if a specified plugin is installed.
*
* @param string $component The component name e.g. tool_log
* @return bool Whether this component is installed.
*/
protected function check_plugin_is_installed(string $component) : Bool {
if (!isset($this->checkedplugins[$component])) {
$pluginmanager = \core_plugin_manager::instance();
$plugin = $pluginmanager->get_plugin_info($component);
$this->checkedplugins[$component] = !is_null($plugin);
}
return $this->checkedplugins[$component];
}
/**
* Writes the appropriate files for creating an HTML index page for human navigation of the user data export.
*/
protected function write_html_data() {
global $PAGE, $SITE, $USER, $CFG;
// Do this first before adding more files to $this->files.
list($tree, $treekey, $allfiles) = $this->prepare_for_export();
// Add more detail to the tree such as contexts.
$richtree = $this->make_tree_object($tree, $treekey);
// Now that we have more detail we can use that to sort it.
$this->sort_my_list($richtree);
// Copy over the JavaScript required to display the html page.
$jspath = ['privacy', 'export_files', 'general.js'];
$targetpath = ['js', 'general.js'];
$this->copy_data($jspath, $targetpath);
$jquery = ['lib', 'jquery', 'jquery-3.2.1.min.js'];
$jquerydestination = ['js', 'jquery-3.2.1.min.js'];
$this->copy_data($jquery, $jquerydestination);
$requirecurrentpath = ['lib', 'requirejs', 'require.min.js'];
$destination = ['js', 'require.min.js'];
$this->copy_data($requirecurrentpath, $destination);
$treepath = ['lib', 'amd', 'build', 'tree.min.js'];
$destination = ['js', 'tree.min.js'];
$this->copy_data($treepath, $destination);
// Icons to be used.
$expandediconpath = ['pix', 't', 'expanded.svg'];
$this->copy_data($expandediconpath, ['pix', 'expanded.svg']);
$collapsediconpath = ['pix', 't', 'collapsed.svg'];
$this->copy_data($collapsediconpath, ['pix', 'collapsed.svg']);
$naviconpath = ['pix', 'i', 'navigationitem.svg'];
$this->copy_data($naviconpath, ['pix', 'navigationitem.svg']);
$moodleimgpath = ['pix', 'moodlelogo.svg'];
$this->copy_data($moodleimgpath, ['pix', 'moodlelogo.svg']);
// Additional required css.
// Determine what direction to show the data export page according to the user preference.
$rtl = right_to_left();
if (!$rtl) {
$bootstrapdestination = 'bootstrap.min.css';
$this->write_url_content('https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css',
$bootstrapdestination);
} else {
$rtldestination = 'rtlbootstrap.min.css';
$this->write_url_content('https://cdn.rtlcss.com/bootstrap/v4.0.0/css/bootstrap.min.css', $rtldestination);
}
$csspath = ['privacy', 'export_files', 'general.css'];
$destination = ['general.css'];
$this->copy_data($csspath, $destination);
// Create an index file that lists all, to be newly created, js files.
$encoded = json_encode($allfiles, JSON_PRETTY_PRINT);
$encoded = 'var user_data_index = ' . $encoded;
$path = 'js' . DIRECTORY_SEPARATOR . 'data_index.js';
$this->write_data($path, $encoded);
$output = $PAGE->get_renderer('core_privacy');
$navigationpage = new \core_privacy\output\exported_navigation_page(current($richtree));
$navigationhtml = $output->render_navigation($navigationpage);
$systemname = $SITE->fullname;
$fullusername = fullname($USER);
$siteurl = $CFG->wwwroot;
// Create custom index.html file.
$htmlpage = new \core_privacy\output\exported_html_page($navigationhtml, $systemname, $fullusername, $rtl, $siteurl);
$outputpage = $output->render_html_page($htmlpage);
$this->write_data('index.html', $outputpage);
}
/**
* Perform any required finalisation steps and return the location of the finalised export.
*
* @return string
*/
public function finalise_content() : string {
$this->write_html_data();
$exportfile = make_request_directory() . '/export.zip';
$fp = get_file_packer();
......@@ -366,4 +695,53 @@ class moodle_content_writer implements content_writer {
return $exportfile;
}
/**
* Creates a multidimensional array out of array elements.
*
* @param array $array Array which items are to be condensed into a multidimensional array.
* @return array The multidimensional array.
*/
protected function condense_array(array $array) : Array {
if (count($array) === 2) {
return [$array[0] => $array[1]];
}
if (isset($array[0])) {
return [$array[0] => $this->condense_array(array_slice($array, 1))];
}
return [];
}
/**
* Get the contents of a file.
*
* @param string $filepath The file path.
* @return string contents of the file.
*/
protected function get_file_content(string $filepath) : String {
$filepointer = fopen($filepath, 'r');
$content = '';
while (!feof($filepointer)) {
$content .= fread($filepointer, filesize($filepath));
}
return $content;
}
/**
* Write url files to the export.
*
* @param string $url Url of the file.
* @param string $path Path to store the file.
*/
protected function write_url_content(string $url, string $path) {
$filepointer = fopen($url, 'r');
$targetpath = $this->path . DIRECTORY_SEPARATOR . $path;
check_dir_exists(dirname($targetpath), true, true);
$status = file_put_contents($targetpath, $filepointer);
if ($status === false) {
// There was an error. Throw an exception to allow the download status to remain as requiring download.
throw new \moodle_exception('Content download was incomplete');
}
$this->files[$path] = $targetpath;
}
}
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Contains the navigation renderable for user data exports.
*
* @package core_privacy
* @copyright 2018 Adrian Greeve
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core_privacy\output;
defined('MOODLE_INTERNAL') || die();
use renderable;
use renderer_base;
use templatable;
/**
* Class containing the navigation renderable
*
* @copyright 2018 Adrian Greeve
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class exported_html_page implements renderable, templatable {
/** @var string $navigationdata navigation html to be displayed about the system. */
protected $navigationdata;
/** @var string $systemname systemname for the page. */
protected $systemname;
/** @var string $username The full name of the user. */
protected $username;
/** @var bool $rtl The direction to show the page (right to left) */
protected $rtl;
/** @var string $siteurl The url back to the site that created this export. */
protected $siteurl;
/**
* Constructor.
*
* @param string $navigationdata Navigation html to be displayed about the system.
* @param string $systemname systemname for the page.
* @param string $username The full name of the user.
* @param bool $righttoleft Is the language used right to left?
* @param string $siteurl The url to the site that created this export.
*/
public function __construct(string $navigationdata, string $systemname, string $username, bool $righttoleft, string $siteurl) {
$this->navigationdata = $navigationdata;
$this->systemname = $systemname;
$this->username = $username;
$this->rtl = $righttoleft;
$this->siteurl = $siteurl;
}
/**
* Export this data so it can be used as the context for a mustache template.
*
* @param renderer_base $output
* @return array
*/
public function export_for_template(renderer_base $output) : Array {
return [
'navigation' => $this->navigationdata,
'systemname' => $this->systemname,
'timegenerated' => time(),
'username' => $this->username,
'righttoleft' => $this->rtl,
'siteurl' => $this->siteurl
];
}
}
\ No newline at end of file
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.
/**
* Contains the navigation renderable for user data exports.
*
* @package core_privacy
* @copyright 2018 Adrian Greeve
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
namespace core_privacy\output;
defined('MOODLE_INTERNAL') || die();
use renderable;
use renderer_base;
use templatable;
/**
* Class containing the navigation renderable
*
* @copyright 2018 Adrian Greeve
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class exported_navigation_page implements renderable, templatable {
/** @var array $tree Full tree in multidimensional form. */
protected $tree;
/** @var boolean $firstelement This is used to create unique classes for the first elements in the navigation tree. */