From 39aac804df175cf469c16dfa94082ff6022e4802 Mon Sep 17 00:00:00 2001 From: Cameron Date: Thu, 21 Jan 2016 13:36:02 -0800 Subject: [PATCH] Fix for corrupted page totals due to SEF URLs and shortened array keys. Requires more testing. BACKUP FIRST! --- e107_handlers/admin_log_class.php | 2 +- e107_plugins/log/admin_config.php | 101 +++++++++++++++++++ e107_plugins/log/consolidate.php | 56 +++++++--- e107_plugins/log/e_shortcode.php | 18 +++- e107_plugins/log/languages/English_admin.php | 4 +- e107_plugins/log/log.php | 16 +-- e107_plugins/log/stats.php | 24 ++++- e107_web/js/core/admin.jquery.js | 5 +- 8 files changed, 195 insertions(+), 31 deletions(-) diff --git a/e107_handlers/admin_log_class.php b/e107_handlers/admin_log_class.php index 3f687c278..e05c62889 100644 --- a/e107_handlers/admin_log_class.php +++ b/e107_handlers/admin_log_class.php @@ -764,7 +764,7 @@ class e_admin_log foreach($this->_allMessages as $m) { - $text .= date('Y-m-d H:i:s', $m['time'])." \t".str_pad($m['dislevel'],10," ",STR_PAD_RIGHT)."\t".strip_tags($m['message'])."\n"; + $text .= date('Y-m-d H:i:s', $m['time'])." \t".str_pad($m['loglevel'],10," ",STR_PAD_RIGHT)."\t".strip_tags($m['message'])."\n"; } $date = ($append == true) ? date('Y-m-d') : date('Y-m-d_H-i-s').'_'.crc32($text); diff --git a/e107_plugins/log/admin_config.php b/e107_plugins/log/admin_config.php index 68475daab..41dd38b49 100644 --- a/e107_plugins/log/admin_config.php +++ b/e107_plugins/log/admin_config.php @@ -18,6 +18,72 @@ if (!getperms('P') || !e107::isInstalled('log')) exit; } + +if(e_AJAX_REQUEST && varset($_GET['action']) == 'rebuild') +{ + require_once(e_PLUGIN."log/consolidate.php"); + $lgc = new logConsolidate; + + $count = $_SESSION['stats_log_files_count']; + $file = $_SESSION['stats_log_files'][$count]['path']; + $totalFiles = $_SESSION['stats_log_files_total'] - 1; + //$process = true; + + $lg = e107::getAdminLog(); +// $lg->addDebug(print_r($logVals, true)); + + + if($_SESSION['stats_log_files'][$count]['complete'] != 1) + { + $_SESSION['stats_log_files'][$count]['complete'] = 1; + // $lg->addSuccess($count."/".$totalFiles."\t".$file." processing", false); + // if($process) + if($lgc->processRawBackupLog($file, true)) + { + + // sleep(3); + $lg->addSuccess($count."/".$totalFiles."\t".$file." processed.", false); + $_SESSION['stats_log_files'][$count]['complete'] = 1; + $_SESSION['stats_log_files_count']++; + } + else + { + $lg->addError($count."/".$totalFiles."\t".$file." failed.", false); + } + } + else + { + $lg->addWarning($count."/".$totalFiles."\t".$file." skipped", false); + $_SESSION['stats_log_files_count']++; + } + + + $totalOutput = round(( $count/ $totalFiles) * 100, 1); + echo $totalOutput; + + + if($totalOutput > 99.7) + { + // echo 100; + if($lgc->collatePageTotalDB()) + { + $lg->addSuccess("Processed All-Time PageTotal", false); + } + else + { + $lg->addError("Failed to Process All-Time PageTotal", false); + } + + $lg->addSuccess("Processing Complete.", false); + + } + + $lg->toFile('SiteStatsUpgrade','Statistics Update Log', true); + + exit; + +} + define('LogFlagFile', 'LogFlag.php'); include_lan(e_PLUGIN.'log/languages/'.e_LANGUAGE.'.php'); @@ -80,6 +146,7 @@ e107::css('inline', 'td.last.options { padding-right:20px } '); 'main/datasets' => array('caption'=> ADSTAT_LAN_63, 'perm' => 'P'), 'main/rempage' => array('caption'=> ADSTAT_LAN_26, 'perm' => 'P'), 'main/history' => array('caption'=> ADSTAT_LAN_69, 'perm' => 'P'), + 'main/rebuild' => array('caption'=>'Rebuild Statistic Summaries', 'perm'=> 'P'), ); @@ -636,6 +703,40 @@ e107::css('inline', 'td.last.options { padding-right:20px } '); } + function rebuildPage() + { + $frm = e107::getForm(); + $mes = e107::getMessage(); + $tp = e107::getParser(); + + $mes->addWarning(ADSTAT_LAN_84); + $text = $frm->open('rebuild'); + + $files = e107::getFile()->get_files(e_LOG."log",'_SiteStats\.log$'); + + // print_a($_SESSION['stats_log_files']); + + $_SESSION['stats_log_files'] = array(); + $_SESSION['stats_log_files_count'] = 0; + + foreach($files as $f) + { + $_SESSION['stats_log_files'][] = array('path'=> $f['fname'], 'complete'=>0); + } + + $_SESSION['stats_log_files_total'] = count($_SESSION['stats_log_files']); + + // $text .= // . " log files have been found. Click the button below to process these files.

"; + $mes->addWarning($tp->lanVars(ADSTAT_LAN_85, $_SESSION['stats_log_files_total'], true)); + $text .= $frm->progressBar('rebuild-progress',0,array("btn-label"=>"Rebuild Stats", 'url'=>e_REQUEST_URI)); + + $text .= $frm->close(); + return $text; + + + } + + function historyPage() { diff --git a/e107_plugins/log/consolidate.php b/e107_plugins/log/consolidate.php index 993ab9968..701ad8440 100644 --- a/e107_plugins/log/consolidate.php +++ b/e107_plugins/log/consolidate.php @@ -436,21 +436,29 @@ class logConsolidate * @param string $err_code * @return bool|mixed|string */ - function getPageKey($url,$logQry=false,$err_code='') + function getPageKey($url,$logQry=false,$err_code='', $lan=null) { $pageDisallow = "cache|file|eself|admin"; $tagRemove = "(\\\)|(\s)|(\')|(\")|(eself)|( )|(\.php)|(\.html)"; // $tagRemove2 = "(\\\)|(\s)|(\')|(\")|(eself)|( )"; - preg_match("#/(.*?)(\?|$)(.*)#si", $url, $match); - $match[1] = isset($match[1]) ? $match[1] : ''; - $pageName = substr($match[1], (strrpos($match[1], "/")+1)); + // preg_match("#/(.*?)(\?|$)(.*)#si", $url, $match); + // $match[1] = isset($match[1]) ? $match[1] : ''; + // $pageName = substr($match[1], (strrpos($match[1], "/")+1)); + + $pageName = str_replace(SITEURL,'',$url); + $pageName = urldecode($pageName); $pageName = preg_replace("/".$tagRemove."/si", "", $pageName); + if($logQry == false) + { + list($pageName,$tmp) = explode("?",$pageName); + } + if($pageName == "") { - return "index"; + $pageName = "index"; } if(preg_match("/".$pageDisallow."/i", $pageName)) @@ -458,17 +466,25 @@ class logConsolidate return false; } - if ($logQry) - { - $pageName .= '+'.$match[3]; // All queries match - } + // if ($logQry) + // { + // $pageName .= '+'.$match[3]; // All queries match + // } $pageName = $err_code.$pageName; // Add the error code at the beginning, so its treated uniquely // filter out any non-utf8 characters which could halt processing. - $pageName = urldecode($pageName); + $pageName = iconv('UTF-8', 'ASCII//IGNORE', $pageName); + $pageName = trim($pageName,' /'); + + + if(!empty($lan)) + { + $pageName .= "|".$lan; + } + return $pageName; } @@ -498,6 +514,7 @@ class logConsolidate $pageTotal = array(); $line = 0; + if ($handle) { while (($buffer = fgets($handle, 4096)) !== false) @@ -510,7 +527,8 @@ class logConsolidate continue; } - $key = $this->getPageKey($vars['eself']); + $lan = varset($vars['lan'],null); + $key = $this->getPageKey($vars['eself'],false,'',$lan); if(empty($key)) { @@ -519,7 +537,7 @@ class logConsolidate if(!isset($pageTotal[$key])) { - $pageTotal[$key] = array('url'=>'', 'ttl'=>0, 'unq'=>0); + $pageTotal[$key] = array('url'=>'', 'ttl'=>0, 'unq'=>0, 'lan'=>''); } $pageTotal[$key]['url'] = $vars['eself']; @@ -538,6 +556,9 @@ class logConsolidate { $pageTotal[$key]['unq'] += 1; } + + $lan = varset($vars['lan'],''); + $pageTotal[$key]['lan'] = $lan; } $line++; @@ -579,7 +600,7 @@ class logConsolidate { $sql = e107::getDb(); - if($sql->select('logstats','log_id',"log_id='".$datestamp."' ")) + if($sql->select('logstats','log_id',"log_id='".$datestamp."' ") && !$sql->select('logstats','log_id',"log_id='bak-".$datestamp."' ")) { $sql->update('logstats', "log_id='bak-".$datestamp."' WHERE log_id='".$datestamp."' "); } @@ -640,14 +661,17 @@ class logConsolidate { if(!empty($val)) { - list($url,$ttl,$unq) = explode("|",$val); - $key = $this->getPageKey($url); + list($url,$ttl,$unq,$lan) = explode("|",$val); + $lan = vartrue($lan,e_LAN); + $key = $this->getPageKey($url,'','', $lan); $thisTotal[$key]['url'] = $url; + $thisTotal[$key]['lan'] = $lan; $thisTotal[$key]['ttlv'] += $ttl; $thisTotal[$key]['unqv'] += $unq; $pageTotal[$key]['url'] = $url; + $pageTotal[$key]['lan'] = $lan; $pageTotal[$key]['ttlv'] += $ttl; $pageTotal[$key]['unqv'] += $unq; } @@ -753,7 +777,7 @@ class logConsolidate foreach($pageInfo as $key => $value) { - $data .= $value['url']."|".$value['ttl']."|".$value['unq'].chr(1); + $data .= $value['url']."|".$value['ttl']."|".$value['unq'].'|'.varset($value['lan'],e_LAN).chr(1); $dailytotal += $value['ttl']; $uniquetotal += $value['unq']; } diff --git a/e107_plugins/log/e_shortcode.php b/e107_plugins/log/e_shortcode.php index b76409dd6..11e9ecd97 100644 --- a/e107_plugins/log/e_shortcode.php +++ b/e107_plugins/log/e_shortcode.php @@ -11,9 +11,14 @@ class log_shortcodes extends e_shortcode { private $dbPageInfo; + private $lgc; function __construct() { + + require_once(e_PLUGIN."log/consolidate.php"); + $this->lgc = new logConsolidate; + $sql = e107::getDB(); $logfile = e_LOG.'logp_'.date('z.Y', time()).'.php'; /* get today's logfile ... */ @@ -70,16 +75,23 @@ class log_shortcodes extends e_shortcode private function getKey($self) { - $base = basename($self); - list($url,$qry) = explode(".",$base, 2); - return $url; + return $this->lgc->getPageKey($self, false, '', e_LAN); + //$base = basename($self); + // list($url,$qry) = explode(".",$base, 2); + // return $url; } function sc_log_pagecounter($parm) { + + //print_a($this->dbPageInfo); + $url = str_replace("www.", "", e_REQUEST_URL); $id = $this->getKey(e_REQUEST_URL); + // print_a("Checking for: ".$id); + + // print_a($this->dbPageInfo); if(isset($this->dbPageInfo[$id]['url']) && ($this->dbPageInfo[$id]['url'] == e_REQUEST_URL || $this->dbPageInfo[$id]['url'] == $url)) { diff --git a/e107_plugins/log/languages/English_admin.php b/e107_plugins/log/languages/English_admin.php index ff7395720..087f3d23d 100644 --- a/e107_plugins/log/languages/English_admin.php +++ b/e107_plugins/log/languages/English_admin.php @@ -93,8 +93,8 @@ define ('ADSTAT_LAN_80', 'The following pages were deleted:'); define ('ADSTAT_LAN_81', 'Statistics cleared:'); define ('ADSTAT_LAN_82', 'Following values now set:'); define ('ADSTAT_LAN_83', 'Following log ID entries removed:'); -define ('ADSTAT_LAN_84', ''); -define ('ADSTAT_LAN_85', ''); +define ('ADSTAT_LAN_84', 'This proceedure will overwrite the log statistic summaries in your database. (raw log files are left unchanged) Once replaced, the data cannot be recovered. Please backup or export your database first'); +define ('ADSTAT_LAN_85', '[x] log files have been found. Click the button below to process these files.'); diff --git a/e107_plugins/log/log.php b/e107_plugins/log/log.php index 7630368ae..68db851b0 100644 --- a/e107_plugins/log/log.php +++ b/e107_plugins/log/log.php @@ -174,7 +174,7 @@ $pageDisallow = "cache|file|eself|admin"; $tagRemove = "(\\\)|(\s)|(\')|(\")|(eself)|( )|(\.php)|(\.html)"; $tagRemove2 = "(\\\)|(\s)|(\')|(\")|(eself)|( )"; - +/* function logGetPageKey($url,$logQry=false,$err_code='') { global $pageDisallow, $tagRemove; @@ -203,12 +203,13 @@ function logGetPageKey($url,$logQry=false,$err_code='') return $pageName; -} +}*/ + +require_once(e_PLUGIN."log/consolidate.php"); +$lgc = new logConsolidate; - - -if(!$pageName = logGetPageKey($self,$logQry,$err_code)) +if(!$pageName = $lgc->getPageKey($self,false,$err_code,e_LAN)) { if(LOG_DEBUG == true) { @@ -217,7 +218,10 @@ if(!$pageName = logGetPageKey($self,$logQry,$err_code)) return; } - +if(LOG_DEBUG == true) +{ + echo "
File: ".$logPfile; +} //$logfp = fopen(e_LOG.'rcvstring.txt', 'a+'); fwrite($logfp, $pageName."\n"); fclose($logfp); diff --git a/e107_plugins/log/stats.php b/e107_plugins/log/stats.php index 2e814b3cf..b9069a0c3 100644 --- a/e107_plugins/log/stats.php +++ b/e107_plugins/log/stats.php @@ -326,6 +326,7 @@ class siteStats protected $order; protected $bar; + protected $plugFolder; protected $filesiteTotal; protected $filesiteUnique; @@ -737,6 +738,10 @@ class siteStats } $this -> bar = (file_exists(THEME.'images/bar.png') ? THEME.'images/bar.png' : e_IMAGE.'generic/bar.png'); + + + $this->plugFolder = e107::getFolder('plugins'); + /* end constructor */ } @@ -805,6 +810,21 @@ class siteStats } + function getLabel($key,$truncate=false) + { + list($url,$language) = explode("|",$key); + + $url = str_replace($this->plugFolder,'',$url); + + if($truncate) + { + return e107::getParser()->text_truncate($url,50); + } + + return $url; + } + + /** @@ -851,7 +871,7 @@ class siteStats if($info['ttl']) { $percentage = round(($info['ttl']/$totalv) * 100, 2); - $text .= "\n ".$key." + $text .= "\n ".$this->getLabel($key)." \n".$this -> bar($percentage, $info['ttl']." [".$info['unq']."]")."\n".$percentage."%\n\n"; } } @@ -927,7 +947,7 @@ class siteStats $text .= " ".($can_delete ? "".ADSTAT_L39." " : "")." - ".$tp->text_truncate($key,50)." + getLabel($key)."\" >".$this->getLabel($key,true)." "; $text .= " ".$this->bar($percentage, $info['ttlv'])." diff --git a/e107_web/js/core/admin.jquery.js b/e107_web/js/core/admin.jquery.js index 318c7067e..2b3899489 100644 --- a/e107_web/js/core/admin.jquery.js +++ b/e107_web/js/core/admin.jquery.js @@ -270,7 +270,10 @@ $(document).ready(function() // alert(data); $("#"+target).css('width', data+'%'); // update the progress bar width */ $("#"+target).html(data+'%'); // display the numeric value */ - + + + data = parseInt(data); + if(data > 99.999) { clearInterval(progresspump);