MDL-62660 tool_dataprivacy: Add ability to expire data requests

Also replaced Completed status with situation specific statuses.
Also improved UX on request pages in line with expiries and the aadditional statuses.

admin/tool/dataprivacy/classes/api.php

@@ -76,7 +76,7 @@ class api {
     /** The request is now being processed. */
     const DATAREQUEST_STATUS_PROCESSING = 4;
 
-    /** Data request completed. */
+    /** Information/other request completed. */
     const DATAREQUEST_STATUS_COMPLETE = 5;
 
     /** Data request cancelled by the user. */
@@ -85,6 +85,15 @@ class api {
     /** Data request rejected by the DPO. */
     const DATAREQUEST_STATUS_REJECTED = 7;
 
+    /** Data request download ready. */
+    const DATAREQUEST_STATUS_DOWNLOAD_READY = 8;
+
+    /** Data request expired. */
+    const DATAREQUEST_STATUS_EXPIRED = 9;
+
+    /** Data delete request completed, account is removed. */
+    const DATAREQUEST_STATUS_DELETED = 10;
+
     /**
      * Determines whether the user can contact the site's Data Protection Officer via Moodle.
      *
@@ -319,6 +328,18 @@ class api {
             }
         }
 
+        // If any are due to expire, expire them and re-fetch updated data.
+        if (empty($statuses)
+                || in_array(self::DATAREQUEST_STATUS_DOWNLOAD_READY, $statuses)
+                || in_array(self::DATAREQUEST_STATUS_EXPIRED, $statuses)) {
+            $expiredrequests = data_request::get_expired_requests($userid);
+
+            if (!empty($expiredrequests)) {
+                data_request::expire($expiredrequests);
+                $results = self::get_data_requests($userid, $statuses, $types, $sort, $offset, $limit);
+            }
+        }
+
         return $results;
     }
 
@@ -400,6 +421,9 @@ class api {
             self::DATAREQUEST_STATUS_COMPLETE,
             self::DATAREQUEST_STATUS_CANCELLED,
             self::DATAREQUEST_STATUS_REJECTED,
+            self::DATAREQUEST_STATUS_DOWNLOAD_READY,
+            self::DATAREQUEST_STATUS_EXPIRED,
+            self::DATAREQUEST_STATUS_DELETED,
         ];
         list($insql, $inparams) = $DB->get_in_or_equal($nonpendingstatuses, SQL_PARAMS_NAMED);
         $select = 'type = :type AND userid = :userid AND status NOT ' . $insql;
@@ -423,6 +447,9 @@ class api {
             self::DATAREQUEST_STATUS_COMPLETE,
             self::DATAREQUEST_STATUS_CANCELLED,
             self::DATAREQUEST_STATUS_REJECTED,
+            self::DATAREQUEST_STATUS_DOWNLOAD_READY,
+            self::DATAREQUEST_STATUS_EXPIRED,
+            self::DATAREQUEST_STATUS_DELETED,
         ];
 
         return !in_array($status, $finalstatuses);

admin/tool/dataprivacy/classes/data_request.php

@@ -85,6 +85,9 @@ class data_request extends persistent {
                     api::DATAREQUEST_STATUS_COMPLETE,
                     api::DATAREQUEST_STATUS_CANCELLED,
                     api::DATAREQUEST_STATUS_REJECTED,
+                    api::DATAREQUEST_STATUS_DOWNLOAD_READY,
+                    api::DATAREQUEST_STATUS_EXPIRED,
+                    api::DATAREQUEST_STATUS_DELETED,
                 ],
                 'type' => PARAM_INT
             ],
@@ -110,4 +113,101 @@ class data_request extends persistent {
             ],
         ];
     }
+
+    /**
+     * Determines whether a completed data export request has expired.
+     * The response will be valid regardless of the expiry scheduled task having run.
+     *
+     * @param data_request $request the data request object whose expiry will be checked.
+     * @return bool true if the request has expired.
+     */
+    public static function is_expired(data_request $request) {
+        $result = false;
+
+        // Only export requests expire.
+        if ($request->get('type') == api::DATAREQUEST_TYPE_EXPORT) {
+            switch ($request->get('status')) {
+                // Expired requests are obviously expired.
+                case api::DATAREQUEST_STATUS_EXPIRED:
+                    $result = true;
+                    break;
+                // Complete requests are expired if the expiry time has elapsed.
+                case api::DATAREQUEST_STATUS_DOWNLOAD_READY:
+                    $expiryseconds = get_config('tool_dataprivacy', 'privacyrequestexpiry');
+                    if ($expiryseconds > 0 && time() >= ($request->get('timemodified') + $expiryseconds)) {
+                        $result = true;
+                    }
+                    break;
+            }
+        }
+
+        return $result;
+    }
+
+
+
+    /**
+     * Fetch completed data requests which are due to expire.
+     *
+     * @param int $userid Optional user ID to filter by.
+     *
+     * @return array Details of completed requests which are due to expire.
+     */
+    public static function get_expired_requests($userid = 0) {
+        global $DB;
+
+        $expiryseconds = get_config('tool_dataprivacy', 'privacyrequestexpiry');
+        $expirytime = strtotime("-{$expiryseconds} second");
+        $table = data_request::TABLE;
+        $sqlwhere = 'type = :export_type AND status = :completestatus AND timemodified <= :expirytime';
+        $params = array(
+            'export_type' => api::DATAREQUEST_TYPE_EXPORT,
+            'completestatus' => api::DATAREQUEST_STATUS_DOWNLOAD_READY,
+            'expirytime' => $expirytime,
+        );
+        $sort = 'id';
+        $fields = 'id, userid';
+
+        // Filter by user ID if specified.
+        if ($userid > 0) {
+            $sqlwhere .= ' AND (userid = :userid OR requestedby = :requestedby)';
+            $params['userid'] = $userid;
+            $params['requestedby'] = $userid;
+        }
+
+        return $DB->get_records_select_menu($table, $sqlwhere, $params, $sort, $fields, 0, 2000);
+    }
+
+    /**
+     * Expire a given set of data requests.
+     * Update request status and delete the files.
+     *
+     * @param array $expiredrequests [requestid => userid]
+     *
+     * @return void
+     */
+    public static function expire($expiredrequests) {
+        global $DB;
+
+        $ids = array_keys($expiredrequests);
+
+        if (count($ids) > 0) {
+            list($insql, $inparams) = $DB->get_in_or_equal($ids);
+            $initialparams = array(api::DATAREQUEST_STATUS_EXPIRED, time());
+            $params = array_merge($initialparams, $inparams);
+
+            $update = "UPDATE {" . data_request::TABLE . "}
+                          SET status = ?, timemodified = ?
+                        WHERE id $insql";
+
+            if ($DB->execute($update, $params)) {
+                $fs = get_file_storage();
+
+                foreach ($expiredrequests as $id => $userid) {
+                    $usercontext = \context_user::instance($userid);
+                    $fs->delete_area_files($usercontext->id, 'tool_dataprivacy', 'export', $id);
+                }
+            }
+        }
+    }
 }

admin/tool/dataprivacy/classes/external/data_request_exporter.php

@@ -160,7 +160,7 @@ class data_request_exporter extends persistent_exporter {
 
         switch ($this->persistent->get('status')) {
             case api::DATAREQUEST_STATUS_PENDING:
-                $values['statuslabelclass'] = 'label-default';
+                $values['statuslabelclass'] = 'label-info';
                 // Request can be manually completed for general enquiry requests.
                 $values['canmarkcomplete'] = $requesttype == api::DATAREQUEST_TYPE_OTHERS;
                 break;
@@ -181,6 +181,8 @@ class data_request_exporter extends persistent_exporter {
                 $values['statuslabelclass'] = 'label-info';
                 break;
             case api::DATAREQUEST_STATUS_COMPLETE:
+            case api::DATAREQUEST_STATUS_DOWNLOAD_READY:
+            case api::DATAREQUEST_STATUS_DELETED:
                 $values['statuslabelclass'] = 'label-success';
                 break;
             case api::DATAREQUEST_STATUS_CANCELLED:
@@ -189,6 +191,9 @@ class data_request_exporter extends persistent_exporter {
             case api::DATAREQUEST_STATUS_REJECTED:
                 $values['statuslabelclass'] = 'label-important';
                 break;
+            case api::DATAREQUEST_STATUS_EXPIRED:
+                $values['statuslabelclass'] = 'label-default';
+                break;
         }
 
         return $values;

admin/tool/dataprivacy/classes/local/helper.php

@@ -117,6 +117,7 @@ class helper {
         if (!isset($statuses[$status])) {
             throw new moodle_exception('errorinvalidrequeststatus', 'tool_dataprivacy');
         }
+
         return $statuses[$status];
     }
 
@@ -133,8 +134,11 @@ class helper {
             api::DATAREQUEST_STATUS_APPROVED => get_string('statusapproved', 'tool_dataprivacy'),
             api::DATAREQUEST_STATUS_PROCESSING => get_string('statusprocessing', 'tool_dataprivacy'),
             api::DATAREQUEST_STATUS_COMPLETE => get_string('statuscomplete', 'tool_dataprivacy'),
+            api::DATAREQUEST_STATUS_DOWNLOAD_READY => get_string('statusready', 'tool_dataprivacy'),
+            api::DATAREQUEST_STATUS_EXPIRED => get_string('statusexpired', 'tool_dataprivacy'),
             api::DATAREQUEST_STATUS_CANCELLED => get_string('statuscancelled', 'tool_dataprivacy'),
             api::DATAREQUEST_STATUS_REJECTED => get_string('statusrejected', 'tool_dataprivacy'),
+            api::DATAREQUEST_STATUS_DELETED => get_string('statusdeleted', 'tool_dataprivacy'),
         ];
     }
 

admin/tool/dataprivacy/classes/output/data_requests_table.php

@@ -59,7 +59,7 @@ class data_requests_table extends table_sql {
     /** @var bool Whether this table is being rendered for managing data requests. */
     protected $manage = false;
 
-    /** @var stdClass[] Array of data request persistents. */
+    /** @var \tool_dataprivacy\data_request[] Array of data request persistents. */
     protected $datarequests = [];
 
     /**
@@ -206,14 +206,14 @@ class data_requests_table extends table_sql {
                 $actiontext = get_string('denyrequest', 'tool_dataprivacy');
                 $actions[] = new action_menu_link_secondary($actionurl, null, $actiontext, $actiondata);
                 break;
-        }
-
-        if ($status == api::DATAREQUEST_STATUS_COMPLETE) {
-            $userid = $data->foruser->id;
-            $usercontext = \context_user::instance($userid, IGNORE_MISSING);
-            if ($usercontext && api::can_download_data_request_for_user($userid, $data->requestedbyuser->id)) {
-                $actions[] = api::get_download_link($usercontext, $requestid);
-            }
+            case api::DATAREQUEST_STATUS_DOWNLOAD_READY:
+                $userid = $data->foruser->id;
+                $usercontext = \context_user::instance($userid, IGNORE_MISSING);
+                // If user has permission to view download link, show relevant action item.
+                if ($usercontext && api::can_download_data_request_for_user($userid, $data->requestedbyuser->id)) {
+                    $actions[] = api::get_download_link($usercontext, $requestid);
+                }
+                break;
         }
 
         $actionsmenu = new action_menu($actions);
@@ -236,19 +236,25 @@ class data_requests_table extends table_sql {
     public function query_db($pagesize, $useinitialsbar = true) {
         global $PAGE;
 
-        // Count data requests from the given conditions.
-        $total = api::get_data_requests_count($this->userid, $this->statuses, $this->types);
-        $this->pagesize($pagesize, $total);
+        // Set dummy page total until we fetch full result set.
+        $this->pagesize($pagesize, $pagesize + 1);
 
         $sort = $this->get_sql_sort();
 
         // Get data requests from the given conditions.
         $datarequests = api::get_data_requests($this->userid, $this->statuses, $this->types, $sort,
                 $this->get_page_start(), $this->get_page_size());
+
+        // Count data requests from the given conditions.
+        $total = api::get_data_requests_count($this->userid, $this->statuses, $this->types);
+        $this->pagesize($pagesize, $total);
+
         $this->rawdata = [];
         $context = \context_system::instance();
         $renderer = $PAGE->get_renderer('tool_dataprivacy');
+
         foreach ($datarequests as $persistent) {
+            $this->datarequests[$persistent->get('id')] = $persistent;
             $exporter = new data_request_exporter($persistent, ['context' => $context]);
             $this->rawdata[] = $exporter->export($renderer);
         }

admin/tool/dataprivacy/classes/output/my_data_requests_page.php

@@ -109,13 +109,29 @@ class my_data_requests_page implements renderable, templatable {
                     $item->statuslabelclass = 'label-success';
                     $item->statuslabel = get_string('statuscomplete', 'tool_dataprivacy');
                     $cancancel = false;
-                    // Show download links only for export-type data requests.
-                    $candownload = $type == api::DATAREQUEST_TYPE_EXPORT;
+                    break;
+                case api::DATAREQUEST_STATUS_DOWNLOAD_READY:
+                    $item->statuslabelclass = 'label-success';
+                    $item->statuslabel = get_string('statusready', 'tool_dataprivacy');
+                    $cancancel = false;
+                    $candownload = true;
+
                     if ($usercontext) {
                         $candownload = api::can_download_data_request_for_user(
                                 $request->get('userid'), $request->get('requestedby'));
                     }
                     break;
+                case api::DATAREQUEST_STATUS_DELETED:
+                    $item->statuslabelclass = 'label-success';
+                    $item->statuslabel = get_string('statusdeleted', 'tool_dataprivacy');
+                    $cancancel = false;
+                    break;
+                case api::DATAREQUEST_STATUS_EXPIRED:
+                    $item->statuslabelclass = 'label-default';
+                    $item->statuslabel = get_string('statusexpired', 'tool_dataprivacy');
+                    $item->statuslabeltitle = get_string('downloadexpireduser', 'tool_dataprivacy');
+                    $cancancel = false;
+                    break;
                 case api::DATAREQUEST_STATUS_CANCELLED:
                 case api::DATAREQUEST_STATUS_REJECTED:
                     $cancancel = false;

admin/tool/dataprivacy/classes/task/process_data_request_task.php

@@ -81,6 +81,7 @@ class process_data_request_task extends adhoc_task {
         // Update the status of this request as pre-processing.
         mtrace('Processing request...');
         api::update_request_status($requestid, api::DATAREQUEST_STATUS_PROCESSING);
+        $completestatus = api::DATAREQUEST_STATUS_COMPLETE;
 
         if ($request->type == api::DATAREQUEST_TYPE_EXPORT) {
             // Get the collection of approved_contextlist objects needed for core_privacy data export.
@@ -115,10 +116,11 @@ class process_data_request_task extends adhoc_task {
             $manager->set_observer(new \tool_dataprivacy\manager_observer());
 
             $manager->delete_data_for_user($approvedclcollection);
+            $completestatus = api::DATAREQUEST_STATUS_DELETED;
         }
 
         // When the preparation of the metadata finishes, update the request status to awaiting approval.
-        api::update_request_status($requestid, api::DATAREQUEST_STATUS_COMPLETE);
+        api::update_request_status($requestid, $completestatus);
         mtrace('The processing of the user data request has been completed...');
 
         // Create message to notify the user regarding the processing results.

admin/tool/dataprivacy/db/upgrade.php

@@ -23,6 +23,7 @@
  */
 
 defined('MOODLE_INTERNAL') || die();
+use tool_dataprivacy\api;
 
 /**
  * Function to upgrade tool_dataprivacy.
@@ -145,5 +146,31 @@ function xmldb_tool_dataprivacy_upgrade($oldversion) {
         upgrade_plugin_savepoint(true, 2018051405, 'tool', 'dataprivacy');
     }
 
+    if ($oldversion < 2018051406) {
+        // Update completed delete requests to new delete status.
+        $query = "UPDATE {tool_dataprivacy_request}
+                     SET status = :setstatus
+                   WHERE type = :type
+                         AND status = :wherestatus";
+        $params = array(
+            'setstatus' => 10, // Request deleted.
+            'type' => 2, // Delete type.
+            'wherestatus' => 5, // Request completed.
+        );
+
+        $DB->execute($query, $params);
+
+        // Update completed data export requests to new download ready status.
+        $params = array(
+            'setstatus' => 8, // Request download ready.
+            'type' => 1, // export type.
+            'wherestatus' => 5, // Request completed.
+        );
+
+        $DB->execute($query, $params);
+
+        upgrade_plugin_savepoint(true, 2018051406, 'tool', 'dataprivacy');
+    }
+
     return true;
 }

admin/tool/dataprivacy/lang/en/tool_dataprivacy.php

@@ -86,6 +86,7 @@ $string['defaultssaved'] = 'Defaults saved';
 $string['deny'] = 'Deny';
 $string['denyrequest'] = 'Deny request';
 $string['download'] = 'Download';
+$string['downloadexpireduser'] = 'Download has expired. Submit a new request if you wish to export your personal data.';
 $string['dporolemapping'] = 'Privacy officer role mapping';
 $string['dporolemapping_desc'] = 'The privacy officer can manage data requests. The capability tool/dataprivacy:managedatarequests must be allowed for a role to be listed as a privacy officer role mapping option.';
 $string['editcategories'] = 'Edit categories';
@@ -192,6 +193,8 @@ $string['privacy:metadata:request:userid'] = 'The ID of the user to whom the req
 $string['privacy:metadata:request:requestedby'] = 'The ID of the user making the request, if made on behalf of another user.';
 $string['privacy:metadata:request:dpocomment'] = 'Any comments made by the site\'s privacy officer regarding the request.';
 $string['privacy:metadata:request:timecreated'] = 'The timestamp indicating when the request was made by the user.';
+$string['privacyrequestexpiry'] = 'Data request expiry';
+$string['privacyrequestexpiry_desc'] = 'The amount of time that approved data requests will be available for download before expiring. 0 means no time limit.';
 $string['protected'] = 'Protected';
 $string['protectedlabel'] = 'The retention of this data has a higher legal precedent over a user\'s request to be forgotten. This data will only be deleted after the retention period has expired.';
 $string['purpose'] = 'Purpose';
@@ -241,7 +244,10 @@ $string['statusapproved'] = 'Approved';
 $string['statusawaitingapproval'] = 'Awaiting approval';
 $string['statuscancelled'] = 'Cancelled';
 $string['statuscomplete'] = 'Complete';
+$string['statusready'] = 'Download ready';
+$string['statusdeleted'] = 'Deleted';
 $string['statusdetail'] = 'Status:';
+$string['statusexpired'] = 'Expired';
 $string['statuspreprocessing'] = 'Pre-processing';
 $string['statusprocessing'] = 'Processing';
 $string['statuspending'] = 'Pending';

admin/tool/dataprivacy/lib.php

@@ -199,6 +199,11 @@ function tool_dataprivacy_pluginfile($course, $cm, $context, $filearea, $args, $
             return false;
         }
 
+        // Make the file unavailable if it has expired.
+        if (\tool_dataprivacy\data_request::is_expired($datarequest)) {
+            send_file_not_found();
+        }
+
         // All good. Serve the exported data.
         $fs = get_file_storage();
         $relativepath = implode('/', $args);

admin/tool/dataprivacy/settings.php

@@ -34,6 +34,12 @@ if ($hassiteconfig) {
                 new lang_string('contactdataprotectionofficer_desc', 'tool_dataprivacy'), 0)
         );
 
+        // Set days approved data requests will be accessible. 1 week default.
+        $privacysettings->add(new admin_setting_configduration('tool_dataprivacy/privacyrequestexpiry',
+                new lang_string('privacyrequestexpiry', 'tool_dataprivacy'),
+                new lang_string('privacyrequestexpiry_desc', 'tool_dataprivacy'),
+                WEEKSECS, 1));
+
         // Fetch roles that are assignable.
         $assignableroles = get_assignable_roles(context_system::instance());
 

admin/tool/dataprivacy/templates/my_data_requests.mustache

@@ -60,7 +60,7 @@
                 "typename" : "Data deletion",
                 "comments": "Please delete all of my son's personal data.",
                 "statuslabelclass": "label-success",
-                "statuslabel": "Complete",
+                "statuslabel": "Deleted",
                 "timecreated" : 1517902087,
                 "requestedbyuser" : {
                     "fullname": "Martha Smith",
@@ -90,6 +90,19 @@
                     "fullname": "Martha Smith",
                     "profileurl": "#"
                 }
+            },
+            {
+                "id": 6,
+                "typename" : "Data export",
+                "comments": "Please let me download my data",
+                "statuslabelclass": "label",
+                "statuslabel": "Expired",
+                "statuslabeltitle": "Download has expired. Submit a new request if you wish to export your personal data.",
+                "timecreated" : 1517902087,
+                "requestedbyuser" : {
+                    "fullname": "Martha Smith",
+                    "profileurl": "#"
+                }
             }
         ]
     }
@@ -127,7 +140,7 @@
                 <td>{{#userdate}} {{timecreated}}, {{#str}} strftimedatetime {{/str}} {{/userdate}}</td>
                 <td><a href="{{requestedbyuser.profileurl}}" title="{{#str}}viewprofile{{/str}}">{{requestedbyuser.fullname}}</a></td>
                 <td>
-                    <span class="label {{statuslabelclass}}">{{statuslabel}}</span>
+                    <span class="label {{statuslabelclass}}" title="{{statuslabeltitle}}">{{statuslabel}}</span>
                 </td>
                 <td>{{comments}}</td>
                 <td>

admin/tool/dataprivacy/version.php

@@ -24,6 +24,6 @@
 
 defined('MOODLE_INTERNAL') || die;
 
-$plugin->version   = 2018051405;
+$plugin->version   = 2018051406;
 $plugin->requires  = 2018050800;        // Moodle 3.5dev (Build 2018031600) and upwards.
 $plugin->component = 'tool_dataprivacy';

privacy/classes/local/request/writer.php

@@ -67,6 +67,24 @@ class writer {
         return $this->realwriter;
     }
 
+    /**
+     * Create a real content_writer for use by PHPUnit tests,
+     * where a mock writer will not suffice.
+     *
+     * @return  content_writer
+     */
+    public static function setup_real_writer_instance() {
+        if (!PHPUNIT_TEST) {
+            throw new coding_exception('setup_real_writer_instance() is only for use with PHPUnit tests.');
+        }
+
+        $instance = static::instance();
+
+        if (null === $instance->realwriter) {
+            $instance->realwriter = new moodle_content_writer(static::instance());
+        }
+    }
+
     /**
      * Return an instance of
      */