From df7ecfe48686aab0bc169ee9c65717d9102a3502 Mon Sep 17 00:00:00 2001 From: skodak Date: Fri, 2 Nov 2007 08:41:05 +0000 Subject: [PATCH] MDL-11996 bulk user upload - improvements, fixes and cleanup - part2 --- admin/uploaduser.php | 355 ++++++++++++++++++++++++++++---------- admin/uploaduser_form.php | 75 ++++++-- lang/en_utf8/admin.php | 2 + 3 files changed, 332 insertions(+), 100 deletions(-) diff --git a/admin/uploaduser.php b/admin/uploaduser.php index 2cc4e6fbc8a..246951b318c 100755 --- a/admin/uploaduser.php +++ b/admin/uploaduser.php @@ -1,29 +1,32 @@ -libdir.'/adminlib.php'); -require_once($CFG->libdir.'/textlib.class.php'); require_once('uploaduser_form.php'); -define('LINE_MAX_SIZE', 1024); -//Note: commas within a field should be encoded as , (for comma separated csv files) -//Note: semicolon within a field should be encoded as ; (for semicolon separated csv files) -$csv_delimiter = isset($CFG->CSV_DELIMITER) ? $CFG->CSV_DELIMITER : ','; -$csv_encode = '&#' . (isset($CFG->CSV_ENCODE) ? $CFG->CSV_ENCODE : ord($csv_delimiter)); +$uplid = optional_param('uplid', '', PARAM_FILE); +$previewrows = optional_param('previewrows', 10, PARAM_INT); +$separator = optional_param('separator', 'comma', PARAM_ALPHA); -@set_time_limit(0); -@raise_memory_limit('192M'); +if (!defined('UP_LINE_MAX_SIZE')) { + define('UP_LINE_MAX_SIZE', 4096); +} + +@set_time_limit(3600); // 1 hour should be enough +@raise_memory_limit('256M'); +if (function_exists('apache_child_terminate')) { + // if we are running from Apache, give httpd a hint that + // it can recycle the process after it's done. Apache's + // memory management is truly awful but we can help it. + @apache_child_terminate(); +} admin_externalpage_setup('uploadusers'); require_capability('moodle/site:uploadusers', get_context_instance(CONTEXT_SYSTEM)); -if (! $site = get_site()) { - error('Could not find site-level course'); -} - $textlib = textlib_get_instance(); $struserrenamed = get_string('userrenamed', 'admin'); @@ -45,85 +48,158 @@ $strcannotassignrole = get_string('cannotassignrole', 'error'); $strduplicateusername = get_string('duplicateusername', 'error'); $strindent = '-->'; -$mform = new admin_uploaduser_form(); +$return = $CFG->wwwroot.'/'.$CFG->admin.'/uploaduser.php'; -// Print the header +// make arrays of valid fields for error checking +// the value associated to each field is: 0 = optional field, 1 = field required either in default values or in data file +$fields = array( + 'firstname' => 1, + 'lastname' => 1, + 'username' => 1, + 'email' => 1, + 'city' => 1, + 'country' => 1, + 'lang' => 1, + 'auth' => 1, + 'timezone' => 1, + 'mailformat' => 1, + 'maildisplay' => 1, + 'htmleditor' => 0, + 'ajax' => 0, + 'autosubscribe' => 1, + 'mnethostid' => 0, + 'institution' => 0, + 'department' => 0, + 'idnumber' => 0, + 'icq' => 0, + 'phone1' => 0, + 'phone2' => 0, + 'address' => 0, + 'url' => 0, + 'description' => 0, + 'icq' => 0, + 'oldusername' => 0, + 'emailstop' => 1, + 'deleted' => 0, + 'password' => 0, // changed later +); -admin_externalpage_print_header(); +if (empty($uplid)) { + $mform = new admin_uploaduser_form1(); + + if ($formdata = $mform->get_data()) { + if (!$filename = make_upload_directory('temp/uploaduser/'.$USER->id, true)) { + error('Can not create temporary upload directory!', $return); + } + // use current (non-conflicting) time stamp + $uplid = time(); + while (file_exists($filename.'/'.$uplid)) { + $uplid--; + } + $filename = $filename.'/'.$uplid; + + $text = $mform->get_file_content('userfile'); + // convert to utf-8 encoding + $text = $textlib->convert($text, $formdata->encoding, 'utf-8'); + // remove Unicode BOM from first line + $text = $textlib->trim_utf8_bom($text); + // Fix mac/dos newlines + $text = preg_replace('!\r\n?!', "\n", $text); + //remove empty lines at the beginning and end + $text = trim($text); + + // verify each line has the same number of separators - this detects major breakage in files + $line = strtok($text, "\n"); + if ($line === false) { + error('Empty file', $return); //TODO: localize + } + + // test headers + $csv_delimiter = get_upload_csv_delimiter($separator); + $col_count = substr_count($line, $csv_delimiter); + if ($col_count < 2) { + error('Not enough columns, please verify the separator setting!', $return); //TODO: localize + } + + $line = explode($csv_delimiter, $line); + foreach ($line as $key => $value) { + $value = trim($value); // remove whitespace + if (!array_key_exists($value, $fields) && // if not a standard field and not an enrolment field, then we have an error + !preg_match('/^course\d+$/', $value) && !preg_match('/^group\d+$/', $value) && + !preg_match('/^type\d+$/', $value) && !preg_match('/^role\d+$/', $value)) { + error(get_string('invalidfieldname', 'error', $value), $return); + } + } + + $line = strtok("\n"); + if ($line === false) { + error('Only one row present, can not continue!', $return); //TODO: localize + } + + while ($line !== false) { + if (substr_count($line, $csv_delimiter) !== $col_count) { + error('Incorrect file format - number of columns is not constant!', $return); //TODO: localize + } + $line = strtok("\n"); + } + + // store file + $fp = fopen($filename, "w"); + fwrite($fp,$text); + fclose($fp); + // continue to second form + + } else { + admin_externalpage_print_header(); + print_heading_with_help(get_string('uploadusers'), 'uploadusers2'); + $mform->display(); + admin_externalpage_print_footer(); + die; + } +} + +$mform = new admin_uploaduser_form2(); +// set initial date from form1 +$mform->set_data(array('separator'=>$separator, 'uplid'=>$uplid, 'previewrows'=>$previewrows)); // If a file has been uploaded, then process it -if ( $formdata = $mform->get_data() ) { +if ($formdata = $mform->is_cancelled()) { + user_upload_cleanup($uplid); + redirect($return); + +} else if ($formdata = $mform->get_data()) { + // Print the header + admin_externalpage_print_header(); + print_heading(get_string('uploadusers')); + $createpassword = $formdata->createpassword; $updateaccounts = $formdata->updateaccounts; $allowrenames = $formdata->allowrenames; $skipduplicates = $formdata->duplicatehandling; - // make arrays of valid fields for error checking - // the value associated to each field is: 0 = optional field, 1 = field required either in default values or in data file - $fields = array( - 'firstname' => 1, - 'lastname' => 1, - 'username' => 1, - 'email' => 1, - 'city' => 1, - 'country' => 1, - 'lang' => 1, - 'auth' => 1, - 'timezone' => 1, - 'mailformat' => 1, - 'maildisplay' => 1, - 'htmleditor' => 0, - 'ajax' => 0, - 'autosubscribe' => 1, - 'mnethostid' => 0, - 'institution' => 0, - 'department' => 0, - 'idnumber' => 0, - 'icq' => 0, - 'phone1' => 0, - 'phone2' => 0, - 'address' => 0, - 'url' => 0, - 'description' => 0, - 'icq' => 0, - 'oldusername' => 0, - 'emailstop' => 1, - 'deleted' => 0, - 'password' => !$createpassword, - ); + $fields['password'] = !$createpassword; - $text = $mform->get_file_content('userfile'); - // convert to utf-8 encoding - $text = $textlib->convert($text, $formdata->encoding, 'utf-8'); - // remove Unicode BOM from first line - $text = $textlib->trim_utf8_bom($text); - // Fix mac/dos newlines - $text = preg_replace('!\r\n?!', "\n", $text); + $filename = $CFG->dataroot.'/temp/uploaduser/'.$USER->id.'/'.$uplid; + if (!file_exists($filename)) { + user_upload_cleanup($uplid); + error('Error reading temporary file!', $return); //TODO: localize + } + if (!$fp = fopen($filename, "r")) { + user_upload_cleanup($uplid); + error('Error reading temporary file!', $return); //TODO: localize + } + + $csv_delimiter = get_upload_csv_delimiter($separator); + $csv_encode = get_upload_csv_encode($csv_delimiter); // find header row $headers = array(); - $linenum = 0; - $line = strtok($text, "\n"); - while ($line !== false) { - $linenum++; - $line = trim($line); - if ($line == '') { - //ignore empty lines - $line = strtok("\n"); - continue; - } - $line = explode($csv_delimiter, $line); - // check for valid field names - foreach ($line as $key => $value) { - $value = trim($value); // remove whitespace - if (!in_array($value, $fields) && // if not a standard field and not an enrolment field, then we have an error - !preg_match('/^course\d+$/', $value) && !preg_match('/^group\d+$/', $value) && - !preg_match('/^type\d+$/', $value) && !preg_match('/^role\d+$/', $value)) { - error(get_string('invalidfieldname', 'error', $value), 'uploaduser.php?sesskey='.$USER->sesskey); - } - $headers[$key] = $value; - } - $line = false; // found header line + $linenum = 1; + $line = explode($csv_delimiter, fgets($fp, UP_LINE_MAX_SIZE)); + + // prepare headers + foreach ($line as $key => $value) { + $headers[$key] = trim($value); } // check that required fields are present or a default value for them exists @@ -155,16 +231,9 @@ if ( $formdata = $mform->get_data() ) { unset($tmp); echo '

'; - $line = strtok("\n"); - while ($line !== false) { + while (!feof($fp)) { $linenum++; - $line = trim($line); - if ($line == '') { - //empty line?? - $line = strtok("\n"); - continue; - } - $line = explode($csv_delimiter, $line); + $line = explode($csv_delimiter, fgets($fp, UP_LINE_MAX_SIZE)); $errors = ''; $user = new object(); // by default, use the local mnet id (this may be changed in the file) @@ -416,8 +485,6 @@ if ( $formdata = $mform->get_data() ) { } } } - //read next line - $line = strtok("\n"); } echo '

'; notify(get_string('userscreated', 'admin') . ': ' . $usersnew); @@ -430,11 +497,119 @@ if ( $formdata = $mform->get_data() ) { } notify(get_string('errors', 'admin') . ': ' . $userserrors); } + fclose($fp); + user_upload_cleanup($uplid); echo '
'; + print_continue($return); + admin_externalpage_print_footer(); + die; } +// Print the header +admin_externalpage_print_header(); + /// Print the form print_heading_with_help(get_string('uploadusers'), 'uploadusers2'); + +/// Print csv file preview +$filename = $CFG->dataroot.'/temp/uploaduser/'.$USER->id.'/'.$uplid; +if (!file_exists($filename)) { + error('Error reading temporary file!', $return); //TODO: localize +} +if (!$fp = fopen($filename, "r")) { + error('Error reading temporary file!', $return); //TODO: localize +} + +$csv_delimiter = get_upload_csv_delimiter($separator); +$csv_encode = get_upload_csv_encode($csv_delimiter); + +$header = explode($csv_delimiter, fgets($fp, UP_LINE_MAX_SIZE)); + +$width = count($header); +$columncount = 0; +$rowcount = 0; +echo ''; +echo ''; +foreach ($header as $h) { + echo ''; +} +echo ''; + +while (!feof($fp) and $rowcount <= $previewrows+1) { + $columncount = 0; + $fields = explode($csv_delimiter, fgets($fp, UP_LINE_MAX_SIZE)); + echo ''; + foreach ($fields as $field) { + echo '';; + } + echo ''; +} +if ($rowcount > $previewrows+1) { + echo ''; + foreach ($fields as $field) { + echo '';; + } +} +echo '
'.trim($h).'
'.trim(str_replace($csv_encode, $csv_delimiter, $field)).'
...
'; +fclose($fp); + $mform->display(); admin_externalpage_print_footer(); +die; + +///////////////////////// +/// Utility functions /// +///////////////////////// + +function user_upload_cleanup($uplid) { + global $USER, $CFG; + if (empty($uplid)) { + return; + } + $filename = $CFG->dataroot.'/temp/uploaduser/'.$USER->id.'/'.$uplid; + if (file_exists($filename)) { + @unlink($filename); + } +} + +function get_uf_headers($uplid, $separator) { + global $USER, $CFG; + + $filename = $CFG->dataroot.'/temp/uploaduser/'.$USER->id.'/'.$uplid; + if (!file_exists($filename)) { + return false; + } + $fp = fopen($filename, "r"); + $line = fgets($fp, 2048); + fclose($fp); + if ($line === false) { + return false; + } + + $csv_delimiter = get_upload_csv_delimiter($separator); + $headers = explode($csv_delimiter, $line); + foreach($headers as $key=>$val) { + $headers[$key] = trim($val); + } + return $headers; +} + +function get_upload_csv_delimiter($separator) { + global $CFG; + + switch ($separator) { + case 'semicolon' : return ';'; + case 'colon' : return ':'; + case 'tab' : return "\t"; + case 'cfg' : return isset($CFG->CSV_DELIMITER) ? $CFG->CSV_DELIMITER : ','; + default : return ','; + } +} + +function get_upload_csv_encode($delimiter) { +//Note: commas within a field should be encoded as , (for comma separated csv files) +//Note: semicolon within a field should be encoded as ; (for semicolon separated csv files) + global $CFG; + return '&#' . (isset($CFG->CSV_ENCODE) ? $CFG->CSV_ENCODE : ord($delimiter)); +} ?> diff --git a/admin/uploaduser_form.php b/admin/uploaduser_form.php index 6edfdbc6a01..140a17685b9 100644 --- a/admin/uploaduser_form.php +++ b/admin/uploaduser_form.php @@ -1,7 +1,46 @@ libdir.'/formslib.php'; -class admin_uploaduser_form extends moodleform { +class admin_uploaduser_form1 extends moodleform { + function definition (){ + global $CFG, $USER; + + $this->set_upload_manager(new upload_manager('userfile', false, false, null, false, 0, true, true, false)); + + $mform =& $this->_form; + + $mform->addElement('header', 'settingsheader', get_string('upload')); + + $mform->addElement('file', 'userfile', get_string('file')); + $mform->addRule('userfile', null, 'required'); + + $choices = array('comma'=>',', 'semicolon'=>';', 'colon'=>':', 'tab'=>'\\t'); + if (isset($CFG->CSV_DELIMITER) and !in_array($CFG->CSV_DELIMITER, $choices)) { + $choices['cfg'] = $CFG->CSV_DELIMITER; + } + $mform->addElement('select', 'separator', get_string('csvseparator', 'admin'), $choices); + if (array_key_exists('cfg', $choices)) { + $mform->setDefault('separator', 'cfg'); + } else if (get_string('listsep') == ';') { + $mform->setDefault('separator', 'semicolon'); + } else { + $mform->setDefault('separator', 'comma'); + } + + $textlib = textlib_get_instance(); + $choices = $textlib->get_encodings(); + $mform->addElement('select', 'encoding', get_string('encoding', 'admin'), $choices); + $mform->setDefault('encoding', 'UTF-8'); + + $choices = array('10'=>10, '20'=>20, '100'=>100, '1000'=>1000, '100000'=>100000); + $mform->addElement('select', 'previewrows', get_string('rowpreviewnum', 'admin'), $choices); + $mform->setType('previewrows', PARAM_INT); + + $this->add_action_buttons(false, get_string('uploadusers')); + } +} + +class admin_uploaduser_form2 extends moodleform { function definition (){ global $CFG, $USER; @@ -13,14 +52,6 @@ class admin_uploaduser_form extends moodleform { // upload settings and file $mform->addElement('header', 'settingsheader', get_string('settings')); - $mform->addElement('file', 'userfile', get_string('file')); - $mform->addRule('userfile', null, 'required'); - - $textlib = textlib_get_instance(); - $choices = $textlib->get_encodings(); - $mform->addElement('select', 'encoding', get_string('encoding', 'admin'), $choices); - $mform->setDefault('encoding', 'UTF-8'); - $choices = array(0 => get_string('infilefield', 'auth'), 1 => get_string('createpasswordifneeded', 'auth')); $mform->addElement('select', 'createpassword', get_string('passwordhandling', 'auth'), $choices); @@ -131,8 +162,32 @@ class admin_uploaduser_form extends moodleform { $mform->setType('address', PARAM_MULTILANG); $mform->setAdvanced('address'); - $this->add_action_buttons(false, get_string('uploadusers')); +// hidden fields + $mform->addElement('hidden', 'uplid'); + $mform->setType('uplid', PARAM_FILE); + + $mform->addElement('hidden', 'separator'); + $mform->setType('separator', PARAM_ALPHA); + + $mform->addElement('hidden', 'previewrows'); + $mform->setType('previewrows', PARAM_ALPHA); + + $this->add_action_buttons(true, get_string('uploadusers')); } + function definition_after_data() { + $mform =& $this->_form; + + $separator = $mform->getElementValue('separator'); + $uplid = $mform->getElementValue('uplid'); + + if ($headers = get_uf_headers($uplid, $separator)) { + foreach ($headers as $header) { + if ($mform->elementExists($header)) { + $mform->removeElement($header); + } + } + } + } } ?> diff --git a/lang/en_utf8/admin.php b/lang/en_utf8/admin.php index f2834b780d7..1635f73cf89 100644 --- a/lang/en_utf8/admin.php +++ b/lang/en_utf8/admin.php @@ -231,6 +231,7 @@ $string['cronerrorclionly'] = 'Sorry, internet access to this page has been disa $string['cronerrorpassword'] = 'Sorry, you have not provided a valid password to access this page'; $string['cronremotepassword'] = 'Cron password for remote access'; $string['cronwarning'] = 'The cron.php maintenance script has not been run for at least 24 hours.'; +$string['csvseparator'] = 'CSV separator'; $string['curlrecommended'] = 'Installing the optional Curl library is highly recommended in order to enable Moodle Networking functionality.'; $string['customcheck'] = 'Other Checks'; $string['datarootsecuritywarning'] = 'Your site configuration might not be secure. Please make sure that your dataroot directory ($a) is not directly accessible via web.'; @@ -547,6 +548,7 @@ $string['riskspam'] = 'Users could send spam to site users or others'; $string['riskspamshort'] = 'Spam risk'; $string['riskxss'] = 'Users could add files and texts that allow cross-site scripting (XSS)'; $string['riskxssshort'] = 'XSS risk'; +$string['rowpreviewnum'] = 'Preview rows'; $string['runclamavonupload'] = 'Use clam AV on uploaded files'; $string['savechanges'] = 'Save Changes'; $string['search'] = 'Search';