mirror_php/moodle
1
0
Fork 0
mirror of https://github.com/moodle/moodle.git synced 2025-04-14 04:52:36 +02:00
Code Issues Projects Releases Wiki Activity

MDL-35773 Backup: API should have option to not backup files

Browse Source 
Allow both UI and automated backups to be created without
including files.  Instead include only file references.
This is essentially implementing "SAMESITE" to backup files
instead of only for import and export functionality.
A new backup setting to include files (defaults to yes)
has been included.

The restore process will also look for and attempt to
restore files from the trashdir as part of restoring
backups.  Additionally to support this process the
ammount of time files are kept in trashdir before they
are cleaned up via cron is also adjustable via admin
setting.
This commit is contained in:
Matt Porritt 2018-12-10 12:27:26 +11:00 committed by Mark Nelson
parent f622ee97e3
commit d7e4481e98
17 changed files with 190 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
admin/settings/courses.php
View File

@ -206,6 +206,11 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) {
$temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_role_assignments', new lang_string('generalroleassignments','backup'), new lang_string('configgeneralroleassignments','backup'), array('value'=>1, 'locked'=>0)));
$temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_activities', new lang_string('generalactivities','backup'), new lang_string('configgeneralactivities','backup'), array('value'=>1, 'locked'=>0)));
$temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_blocks', new lang_string('generalblocks','backup'), new lang_string('configgeneralblocks','backup'), array('value'=>1, 'locked'=>0)));
$temp->add(new admin_setting_configcheckbox_with_lock(
'backup/backup_general_files',
new lang_string('generalfiles', 'backup'),
new lang_string('configgeneralfiles', 'backup'),
array('value' => '1', 'locked' => 0)));
$temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_filters', new lang_string('generalfilters','backup'), new lang_string('configgeneralfilters','backup'), array('value'=>1, 'locked'=>0)));
$temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_comments', new lang_string('generalcomments','backup'), new lang_string('configgeneralcomments','backup'), array('value'=>1, 'locked'=>0)));
$temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_badges', new lang_string('generalbadges','backup'), new lang_string('configgeneralbadges','backup'), array('value'=>1,'locked'=>0)));
@ -342,6 +347,10 @@ if ($hassiteconfig or has_any_capability($capabilities, $systemcontext)) {
$temp->add(new admin_setting_configcheckbox('backup/backup_auto_activities', new lang_string('generalactivities','backup'), new lang_string('configgeneralactivities','backup'), 1));
$temp->add(new admin_setting_configcheckbox('backup/backup_auto_blocks', new lang_string('generalblocks','backup'), new lang_string('configgeneralblocks','backup'), 1));
$temp->add(new admin_setting_configcheckbox('backup/backup_auto_filters', new lang_string('generalfilters','backup'), new lang_string('configgeneralfilters','backup'), 1));
$temp->add(new admin_setting_configcheckbox(
'backup/backup_auto_files',
new lang_string('generalfiles', 'backup'),
new lang_string('configgeneralfiles', 'backup'), '1'));
$temp->add(new admin_setting_configcheckbox('backup/backup_auto_comments', new lang_string('generalcomments','backup'), new lang_string('configgeneralcomments','backup'), 1));
$temp->add(new admin_setting_configcheckbox('backup/backup_auto_badges', new lang_string('generalbadges','backup'), new lang_string('configgeneralbadges','backup'), 1));
$temp->add(new admin_setting_configcheckbox('backup/backup_auto_calendarevents', new lang_string('generalcalendarevents','backup'), new lang_string('configgeneralcalendarevents','backup'), 1));

5
admin/settings/server.php
View File

@ -172,7 +172,10 @@ $temp->add(new admin_setting_configselect('tempdatafoldercleanup', new lang_stri
$ADMIN->add('server', $temp);
$temp->add(new admin_setting_configduration('filescleanupperiod',
new lang_string('filescleanupperiod', 'admin'),
new lang_string('filescleanupperiod_help', 'admin'),
86400));
$ADMIN->add('server', new admin_externalpage('environment', new lang_string('environment','admin'), "$CFG->wwwroot/$CFG->admin/environment.php"));
$ADMIN->add('server', new admin_externalpage('phpinfo', new lang_string('phpinfo'), "$CFG->wwwroot/$CFG->admin/phpinfo.php"));

10
backup/backup.php
View File

@ -150,6 +150,13 @@ if (!async_helper::is_async_pending($id, 'course', 'backup')) {
$loghtml = '';
if ($backup->get_stage() == backup_ui::STAGE_FINAL) {
// Before we perform the backup check settings to see if user
// or setting defaults are set to exclude files from the backup.
if ($backup->get_setting_value('files') == 0) {
$bc->set_mode(backup::MODE_SAMESITE);
$renderer->set_samesite_notification();
}
if ($backupmode != backup::MODE_ASYNC) {
// Synchronous backup handling.
@ -180,6 +187,7 @@ if (!async_helper::is_async_pending($id, 'course', 'backup')) {
// Hide the progress display and first backup step bar (the 'finished' step will show next).
echo html_writer::end_div();
echo html_writer::script('document.getElementById("executionprogress").style.display = "none";');
} else {
// Async backup handling.
$backup->get_controller()->finish_ui();
@ -203,6 +211,8 @@ if (!async_helper::is_async_pending($id, 'course', 'backup')) {
'restoreurl' => $restoreurl->out(),
'headingident' => 'backup'
);
echo $renderer->set_samesite_notification();
echo $renderer->render_from_template('core/async_backup_status', $progresssetup);
}

21
backup/controller/backup_controller.class.php
View File

@ -185,6 +185,20 @@ class backup_controller extends base_controller {
backup_check::check_security($this, false);
}
/**
* Sets the mode (purpose) of the backup.
*
* @param int $mode The mode to set.
*/
public function set_mode($mode) {
$this->mode = $mode;
$this->set_include_files(); // Need to check if files are included as mode may have changed.
$this->save_controller();
$tbc = self::load_controller($this->backupid);
$this->logger = $tbc->logger; // Wakeup loggers.
$tbc->plan->destroy(); // Clean plan controller structures, keeping logger alive.
}
public function set_status($status) {
// Note: never save_controller() with the object info after STATUS_EXECUTING or the whole controller,
// containing all the steps will be sent to DB. 100% (monster) useless.
@ -410,6 +424,13 @@ class backup_controller extends base_controller {
$includefiles = false;
}
// If backup is automated and we have set auto backup config to exclude
// files then set them to be excluded here.
$backupautofiles = (bool)get_config('backup', 'backup_auto_files');
if ($this->get_mode() === backup::MODE_AUTOMATED && !$backupautofiles) {
$includefiles = false;
}
$this->includefiles = (int) $includefiles;
$this->log("setting file inclusion to {$this->includefiles}", backup::LOG_DEBUG);
return $this->includefiles;

6
backup/moodle2/backup_root_task.class.php
View File

@ -110,6 +110,12 @@ class backup_root_task extends backup_task {
$this->add_setting($blocks);
$this->converter_deps($blocks, $converters);
// Define files.
$files = new backup_generic_setting('files', base_setting::IS_BOOLEAN, true);
$files->set_ui(new backup_setting_ui_checkbox($files, get_string('rootsettingfiles', 'backup')));
$this->add_setting($files);
$this->converter_deps($files, $converters);
// Define filters
$filters = new backup_generic_setting('filters', base_setting::IS_BOOLEAN, true);
$filters->set_ui(new backup_setting_ui_checkbox($filters, get_string('rootsettingfilters', 'backup')));

8
backup/util/dbops/backup_plan_dbops.class.php
View File

@ -201,7 +201,8 @@ abstract class backup_plan_dbops extends backup_dbops {
* @param bool $useidonly only use the ID in the file name
* @return string The filename to use
*/
public static function get_default_backup_filename($format, $type, $id, $users, $anonymised, $useidonly = false) {
public static function get_default_backup_filename($format, $type, $id, $users, $anonymised,
$useidonly = false, $files = true) {
global $DB;
// Calculate backup word
@ -251,6 +252,11 @@ abstract class backup_plan_dbops extends backup_dbops {
$info = '-an';
}
// Indicate if backup doesn't contain files.
if (!$files) {
$info .= '-nf';
}
return $backupword . '-' . $format . '-' . $type . '-' .
$name . '-' . $date . $info . '.mbz';
}

20
backup/util/dbops/restore_dbops.class.php
View File

@ -1054,17 +1054,25 @@ abstract class restore_dbops {
// Create the file in the filepool if it does not exist yet.
if (!$fs->file_exists($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $file->filename)) {
// Even if a file has been deleted since the backup was made, the file metadata will remain in the
// files table, and the file will not be moved to the trashdir.
// Files are not cleared from the files table by cron until several days after deletion.
// Even if a file has been deleted since the backup was made, the file metadata may remain in the
// files table, and the file will not yet have been moved to the trashdir. e.g. a draft file version.
// Try to recover from file table first.
if ($foundfiles = $DB->get_records('files', array('contenthash' => $file->contenthash), '', '*', 0, 1)) {
// Only grab one of the foundfiles - the file content should be the same for all entries.
$foundfile = reset($foundfiles);
$fs->create_file_from_storedfile($file_record, $foundfile->id);
} else {
// A matching existing file record was not found in the database.
$results[] = self::get_missing_file_result($file);
continue;
// Finally try to restore the file from trash.
$filesytem = $fs->get_file_system();
$restorefile = $file;
$restorefile->contextid = $newcontextid;
$storedfile = new stored_file($fs, $restorefile);
$trashrecovery = $filesytem->recover_file($storedfile, true);
if (!$trashrecovery) {
// A matching file was not found.
$results[] = self::get_missing_file_result($file);
continue;
}
}
}
}

3
backup/util/helper/backup_cron_helper.class.php
View File

@ -415,8 +415,9 @@ abstract class backup_cron_automated_helper {
$id = $bc->get_id();
$users = $bc->get_plan()->get_setting('users')->get_value();
$anonymised = $bc->get_plan()->get_setting('anonymize')->get_value();
$incfiles = (bool)$config->backup_auto_files;
$bc->get_plan()->get_setting('filename')->set_value(backup_plan_dbops::get_default_backup_filename($format, $type,
$id, $users, $anonymised));
$id, $users, $anonymised, false, $incfiles));
$bc->set_status(backup::STATUS_AWAITING);

10
backup/util/helper/backup_helper.class.php
View File

@ -287,7 +287,15 @@ abstract class backup_helper {
$config = get_config('backup');
$dir = $config->backup_auto_destination;
if ($config->backup_auto_storage == 1 and $dir and is_dir($dir) and is_writable($dir)) {
$filedest = $dir.'/'.backup_plan_dbops::get_default_backup_filename($format, $backuptype, $courseid, $hasusers, $isannon, !$config->backup_shortname);
$filedest = $dir.'/'
.backup_plan_dbops::get_default_backup_filename(
$format,
$backuptype,
$courseid,
$hasusers,
$isannon,
!$config->backup_shortname,
(bool)$config->backup_auto_files);
// first try to move the file, if it is not possible copy and delete instead
if (@rename($filepath, $filedest)) {
return null;

16
backup/util/ui/backup_ui_stage.class.php
View File

@ -155,7 +155,9 @@ class backup_ui_stage_initial extends backup_ui_stage {
$this->ui->get_type(),
$this->ui->get_controller_id(),
$this->ui->get_setting_value('users'),
$this->ui->get_setting_value('anonymize')
$this->ui->get_setting_value('anonymize'),
false,
(bool)$this->ui->get_setting_value('files')
);
$setting->set_value($filename);
}
@ -457,7 +459,16 @@ class backup_ui_stage_confirmation extends backup_ui_stage {
$id = $this->ui->get_controller_id();
$users = $this->ui->get_setting_value('users');
$anonymised = $this->ui->get_setting_value('anonymize');
$setting->set_value(backup_plan_dbops::get_default_backup_filename($format, $type, $id, $users, $anonymised));
$files = (bool)$this->ui->get_setting_value('files');
$filename = backup_plan_dbops::get_default_backup_filename(
$format,
$type,
$id,
$users,
$anonymised,
false,
$files);
$setting->set_value($filename);
}
$form->add_setting($setting, $task);
break;
@ -628,6 +639,7 @@ class backup_ui_stage_complete extends backup_ui_stage_final {
if (!empty($this->results['missing_files_in_pool'])) {
$output .= $renderer->notification(get_string('missingfilesinpool', 'backup'), 'notifyproblem');
}
$output .= $renderer->get_samesite_notification();
$output .= $renderer->notification(get_string('executionsuccess', 'backup'), 'notifysuccess');
$output .= $renderer->continue_button($restorerul);
$output .= $renderer->box_end();

23
backup/util/ui/renderer.php
View File

@ -43,6 +43,13 @@ require_once($CFG->dirroot . '/backup/moodle2/backup_plan_builder.class.php');
*/
class core_backup_renderer extends plugin_renderer_base {
/**
* Same site notification display.
*
* @var string
*/
private $samesitenotification = '';
/**
* Renderers a progress bar for the backup or restore given the items that make it up.
*
@ -80,6 +87,22 @@ class core_backup_renderer extends plugin_renderer_base {
return $out;
}
/**
* Set the same site backup notification.
*
*/
public function set_samesite_notification() {
$this->samesitenotification = $this->output->notification(get_string('samesitenotification', 'backup'), 'info');
}
/**
* Get the same site backup notification.
*
*/
public function get_samesite_notification() {
return $this->samesitenotification;
}
/**
* Prints a dependency notification
*

2
lang/en/admin.php
View File

@ -564,6 +564,8 @@ $string['experimentalsettings'] = 'Experimental settings';
$string['extendedusernamechars'] = 'Allow extended characters in usernames';
$string['extramemorylimit'] = 'Extra PHP memory limit';
$string['fatalsessionautostart'] = '<p>Serious configuration error detected, please notify server administrator.</p><p> To operate properly, Moodle requires that administrator changes PHP settings.</p><p><code>session.auto_start</code> must be set to <code>off</code>.</p><p>This setting is controlled by editing <code>php.ini</code>, Apache/IIS <br />configuration or <code>.htaccess</code> file on the server.</p>';
$string['filescleanupperiod'] = 'Clean trash pool files';
$string['filescleanupperiod_help'] = 'How often trash files are removed. These are files that are associated with a context that no longer exists';
$string['fileconversioncleanuptask'] = 'Cleanup of temporary records for file conversions.';
$string['filecreated'] = 'New file created';
$string['filestoredin'] = 'Save file into folder :';

4
lang/en/backup.php
View File

@ -127,6 +127,7 @@ $string['configgeneralblocks'] = 'Sets the default for including blocks in a bac
$string['configgeneralcalendarevents'] = 'Sets the default for including calendar events in a backup.';
$string['configgeneralcomments'] = 'Sets the default for including comments in a backup.';
$string['configgeneralcompetencies'] = 'Sets the default for including competencies in a backup.';
$string['configgeneralfiles'] = 'Sets the default for including files in a backup.';
$string['configgeneralfilters'] = 'Sets the default for including filters in a backup.';
$string['configgeneralhistories'] = 'Sets the default for including user history within a backup.';
$string['configgenerallogs'] = 'If enabled logs will be included in backups by default.';
@ -202,6 +203,7 @@ $string['generalcalendarevents'] = 'Include calendar events';
$string['generalcomments'] = 'Include comments';
$string['generalcompetencies'] = 'Include competencies';
$string['generalenrolments'] = 'Include enrolment methods';
$string['generalfiles'] = 'Include files';
$string['generalfilters'] = 'Include filters';
$string['generalhistories'] = 'Include histories';
$string['generalgradehistories'] = 'Include histories';
@ -328,6 +330,7 @@ $string['rootsettingbadges'] = 'Include badges';
$string['rootsettingblocks'] = 'Include blocks';
$string['rootsettingcompetencies'] = 'Include competencies';
$string['rootsettingfilters'] = 'Include filters';
$string['rootsettingfiles'] = 'Include files';
$string['rootsettingcomments'] = 'Include comments';
$string['rootsettingcalendarevents'] = 'Include calendar events';
$string['rootsettinguserscompletion'] = 'Include user completion details';
@ -337,6 +340,7 @@ $string['rootsettinggradehistories'] = 'Include grade history';
$string['rootsettinggroups'] = 'Include groups and groupings';
$string['rootsettingimscc1'] = 'Convert to IMS Common Cartridge 1.0';
$string['rootsettingimscc11'] = 'Convert to IMS Common Cartridge 1.1';
$string['samesitenotification'] = 'This backup was created with only references to files, not the files themselves. Restoring will only work on this site.';
$string['sitecourseformatwarning'] = 'This is a front page backup, note that they can only be restored on the front page';
$string['storagecourseonly'] = 'Course backup filearea';
$string['storagecourseandexternal'] = 'Course backup filearea and the specified directory';

3
lib/filestorage/file_storage.php
View File

@ -2241,7 +2241,8 @@ class file_storage {
// remove trash pool files once a day
// if you want to disable purging of trash put $CFG->fileslastcleanup=time(); into config.php
if (empty($CFG->fileslastcleanup) or $CFG->fileslastcleanup < time() - 60*60*24) {
$filescleanupperiod = empty($CFG->filescleanupperiod) ? 86400 : $CFG->filescleanupperiod;
if (empty($CFG->fileslastcleanup) || ($CFG->fileslastcleanup < time() - $filescleanupperiod)) {
require_once($CFG->libdir.'/filelib.php');
// Delete files that are associated with a context that no longer exists.
mtrace('Cleaning up files from deleted contexts... ', '');

9
lib/filestorage/file_system.php
View File

@ -236,6 +236,15 @@ abstract class file_system {
*/
abstract public function remove_file($contenthash);
/**
* Tries to recover missing content of file from trash.
*
* @param stored_file $file stored_file instance
* @param bool $createrecord Create file record for stored file.
* @return bool success
*/
abstract public function recover_file(stored_file $file, $createrecord=false);
/**
* Check whether a file is removable.
*

57
lib/filestorage/file_system_filedir.php
View File

@ -234,13 +234,46 @@ class file_system_filedir extends file_system {
return copy($source, $target);
}
/**
* Create a file record from a stored file object.
*
* This is required in cases where we are recoverying a file
* from the trash and we need to also recrete the file record
* in the database.
*
* @param stored_file $file
* @return stdClass
*/
protected function create_recovery_record(stored_file $file) {
$filerecord = new stdClass();
$filerecord->contextid = $file->get_contextid();
$filerecord->component = $file->get_component();
$filerecord->filearea = $file->get_filearea();
$filerecord->itemid = $file->get_itemid();
$filerecord->filepath = $file->get_filepath();
$filerecord->filename = $file->get_filename();
$filerecord->timecreated = $file->get_timecreated();
$filerecord->timemodified = $file->get_timemodified();
$filerecord->userid = empty($file->get_userid()) ? null : $file->get_userid();
$filerecord->source = empty($file->get_source()) ? null : $file->get_source();
$filerecord->author = empty($file->get_author()) ? null : $file->get_author();
$filerecord->license = empty($file->get_license()) ? null : $file->get_license();
$filerecord->status = empty($file->get_status()) ? 0 : $file->get_status();
$filerecord->sortorder = $file->get_sortorder();
$filerecord->contenthash = $file->get_contenthash();
return $filerecord;
}
/**
* Tries to recover missing content of file from trash.
*
* @param stored_file $file stored_file instance
* @param bool $createrecord Create file record for stored file.
* @return bool success
*/
protected function recover_file(stored_file $file) {
public function recover_file(stored_file $file, $createrecord=false) {
$contentfile = $this->get_local_path_from_storedfile($file, false);
if (file_exists($contentfile)) {
@ -274,9 +307,25 @@ class file_system_filedir extends file_system {
}
}
// Perform a rename - these are generally atomic which gives us big
// performance wins, especially for large files.
return rename($trashfile, $contentfile);
// Restore file from trash and create file record in database if needed.
if ($createrecord) {
$recoveryrecord = $this->create_recovery_record($file);
$fs = new file_storage();
$fs->create_file_from_pathname($recoveryrecord, $trashfile);
// Remove copy of file still in trash.
// There are no references to this file anywhere so we just unlink it.
unlink($trashfile);
} else {
// If record exists in database then perform a rename.
// These are generally atomic which gives us big
// performance wins, especially for large files.
return rename($trashfile, $contentfile);
}
return true;
}
/**

2
version.php
View File

@ -29,7 +29,7 @@
defined('MOODLE_INTERNAL') || die();
$version = 2019072500.00; // YYYYMMDD = weekly release date of this DEV branch.
$version = 2019072600.00; // YYYYMMDD = weekly release date of this DEV branch.
// RR = release increments - 00 in DEV branches.
// .XX = incremental changes.