Updates to cache CRC values and folder traversal lookup

source/glest_game/network/network_message.cpp

@@ -349,20 +349,24 @@ NetworkMessageSynchNetworkGameData::NetworkMessageSynchNetworkGameData(const Gam
         SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] gameSettings.getScenarioDir() = [%s] gameSettings.getScenario() = [%s] scenarioDir = [%s]\n",__FILE__,__FUNCTION__,__LINE__,gameSettings->getScenarioDir().c_str(),gameSettings->getScenario().c_str(),scenarioDir.c_str());
     }
 
+    SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__);
     //Checksum checksum;
     //data.tilesetCRC = getFolderTreeContentsCheckSumRecursively(string(GameConstants::folder_path_tilesets) + "/" + gameSettings->getTileset() + "/*", "xml", NULL);
 	data.tilesetCRC = getFolderTreeContentsCheckSumRecursively(config.getPathListForType(ptTilesets,scenarioDir), string("/") + gameSettings->getTileset() + string("/*"), ".xml", NULL);
 
+	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__);
     //tech, load before map because of resources
     //data.techCRC  = getFolderTreeContentsCheckSumRecursively(string(GameConstants::folder_path_techs) + "/" + gameSettings->getTech() + "/*", "xml", NULL);
 	data.techCRC = getFolderTreeContentsCheckSumRecursively(config.getPathListForType(ptTechs,scenarioDir), string("/") + gameSettings->getTech() + string("/*"), ".xml", NULL);
 
+	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__);
     //map
     Checksum checksum;
     string file = Map::getMapPath(gameSettings->getMap(),scenarioDir);
 	checksum.addFile(file);
 	data.mapCRC = checksum.getSum();
 	//SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s] file = [%s] checksum = %d\n",__FILE__,__FUNCTION__,file.c_str(),data.mapCRC);
+	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__);
 }
 
 bool NetworkMessageSynchNetworkGameData::receive(Socket* socket)

source/glest_game/network/server_interface.cpp

@@ -531,7 +531,7 @@ void ServerInterface::updateListen()
 
 void ServerInterface::setGameSettings(GameSettings *serverGameSettings, bool waitForClientAck)
 {
-    SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s] START gameSettingsUpdateCount = %d\n",__FILE__,__FUNCTION__,gameSettingsUpdateCount);
+    SystemFlags::OutputDebug(SystemFlags::debugNetwork,"In [%s::%s] START gameSettingsUpdateCount = %d, waitForClientAck = %d\n",__FILE__,__FUNCTION__,gameSettingsUpdateCount,waitForClientAck);
 
     if(getAllowGameDataSynchCheck() == true)
     {

source/shared_lib/include/util/checksum.h

@@ -1,7 +1,7 @@
 // ==============================================================
 //	This file is part of Glest Shared Library (www.glest.org)
 //
-//	Copyright (C) 2001-2008 Marti<EFBFBD>o Figueroa
+//	Copyright (C) 2001-2008 Martio Figueroa
 //
 //	You can redistribute this code and/or modify it under
 //	the terms of the GNU General Public License as published
@@ -34,13 +34,16 @@ private:
     int32	c1;
     int32	c2;
 	std::map<string,int32> fileList;
+	static std::map<string,int32> fileListCache;
 
+	void addSum(int32 value);
 	void addFileToSum(const string &path);
 
 public:
 	Checksum();
 
 	int32 getSum();
+	int32 getFinalFileListSum();
 
 	void addByte(int8 value);
 	void addString(const string &value);

source/shared_lib/sources/platform/sdl/platform_util.cpp

@@ -254,7 +254,7 @@ bool isdir(const char *path)
   struct stat stats;
 
   bool ret = stat (path, &stats) == 0 && S_ISDIR(stats.st_mode);
-  if(ret == false) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] NOT a path [%s]\n",__FILE__,__FUNCTION__,__LINE__,path);
+  //if(ret == false) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] NOT a path [%s]\n",__FILE__,__FUNCTION__,__LINE__,path);
 
   return ret;
 }
@@ -285,21 +285,26 @@ int32 getFolderTreeContentsCheckSumRecursively(vector<string> paths, string path
 		getFolderTreeContentsCheckSumRecursively(path, filterFileExt, &checksum);
 	}
 
-	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] returning: %d\n",__FILE__,__FUNCTION__,__LINE__,checksum.getSum());
+	//SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] returning: %d\n",__FILE__,__FUNCTION__,__LINE__,checksum.getSum());
 
 	if(recursiveChecksum != NULL) {
 		*recursiveChecksum = checksum;
 	}
 
-	return checksum.getSum();
+	return checksum.getFinalFileListSum();
 }
 
 //finds all filenames like path and gets their checksum of all files combined
 int32 getFolderTreeContentsCheckSumRecursively(const string &path, const string &filterFileExt, Checksum *recursiveChecksum) {
 
+	string cacheKey = path + "_" + filterFileExt;
+	static std::map<string,int32> crcTreeCache;
+	if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) {
+		return crcTreeCache[cacheKey];
+	}
     Checksum checksum = (recursiveChecksum == NULL ? Checksum() : *recursiveChecksum);
 
-    SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] starting checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),checksum.getSum());
+    //SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] starting checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),checksum.getSum());
 
 	std::string mypath = path;
 	/** Stupid win32 is searching for all files without extension when *. is
@@ -375,13 +380,17 @@ int32 getFolderTreeContentsCheckSumRecursively(const string &path, const string
 
 	globfree(&globbuf);
 
-	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] ending checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),checksum.getSum());
+	//SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] ending checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),checksum.getSum());
 
 	if(recursiveChecksum != NULL) {
 		*recursiveChecksum = checksum;
 	}
 
-    return checksum.getSum();
+	crcTreeCache[cacheKey] = checksum.getFinalFileListSum();
+
+	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] ending checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),crcTreeCache[cacheKey]);
+
+    return crcTreeCache[cacheKey];
 }
 
 vector<std::pair<string,int32> > getFolderTreeContentsCheckSumListRecursively(vector<string> paths, string pathSearchString, string filterFileExt, vector<std::pair<string,int32> > *recursiveMap) {
@@ -402,6 +411,12 @@ vector<std::pair<string,int32> > getFolderTreeContentsCheckSumListRecursively(ve
 //finds all filenames like path and gets the checksum of each file
 vector<std::pair<string,int32> > getFolderTreeContentsCheckSumListRecursively(const string &path, const string &filterFileExt, vector<std::pair<string,int32> > *recursiveMap) {
 
+	string cacheKey = path + "_" + filterFileExt;
+	static std::map<string,vector<std::pair<string,int32> > > crcTreeCache;
+	if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) {
+		return crcTreeCache[cacheKey];
+	}
+
     vector<std::pair<string,int32> > checksumFiles = (recursiveMap == NULL ? vector<std::pair<string,int32> >() : *recursiveMap);
 
     SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s]\n",__FILE__,__FUNCTION__,path.c_str());
@@ -483,7 +498,9 @@ vector<std::pair<string,int32> > getFolderTreeContentsCheckSumListRecursively(co
 
 	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s]\n",__FILE__,__FUNCTION__,path.c_str());
 
-    return checksumFiles;
+	crcTreeCache[cacheKey] = checksumFiles;
+
+    return crcTreeCache[cacheKey];
 }
 
 string extractDirectoryPathFromFile(string filename)

source/shared_lib/sources/platform/win32/platform_util.cpp

@@ -314,8 +314,9 @@ void findAll(const string &path, vector<string> &results, bool cutExtension, boo
 bool isdir(const char *path)
 {
   struct stat stats;
+
   bool ret = stat (path, &stats) == 0 && S_ISDIR(stats.st_mode);
-  if(ret == false) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] NOT a path [%s]\n",__FILE__,__FUNCTION__,__LINE__,path);
+  //if(ret == false) SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] NOT a path [%s]\n",__FILE__,__FUNCTION__,__LINE__,path);
 
   return ret;
 }
@@ -333,27 +334,39 @@ bool EndsWith(const string &str, const string& key)
 
 //finds all filenames like path and gets their checksum of all files combined
 int32 getFolderTreeContentsCheckSumRecursively(vector<string> paths, string pathSearchString, const string filterFileExt, Checksum *recursiveChecksum) {
+
+	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__);
+
 	Checksum checksum = (recursiveChecksum == NULL ? Checksum() : *recursiveChecksum);
-	size_t count = paths.size();
+	int count = paths.size();
-	for(unsigned int idx = 0; idx < count; ++idx) {
+	for(int idx = 0; idx < count; ++idx) {
 		string path = paths[idx] + pathSearchString;
+
+		SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] path = [%s], filterFileExt = [%s]\n",__FILE__,__FUNCTION__,__LINE__,path.c_str(),filterFileExt.c_str());
+
 		getFolderTreeContentsCheckSumRecursively(path, filterFileExt, &checksum);
 	}
 
+	//SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] returning: %d\n",__FILE__,__FUNCTION__,__LINE__,checksum.getSum());
+
 	if(recursiveChecksum != NULL) {
 		*recursiveChecksum = checksum;
 	}
 
-	return checksum.getSum();
+	return checksum.getFinalFileListSum();
 }
 
 //finds all filenames like path and gets their checksum of all files combined
 int32 getFolderTreeContentsCheckSumRecursively(const string &path, const string &filterFileExt, Checksum *recursiveChecksum) {
 
-
+	string cacheKey = path + "_" + filterFileExt;
+	static std::map<string,int32> crcTreeCache;
+	if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) {
+		return crcTreeCache[cacheKey];
+	}
     Checksum checksum = (recursiveChecksum == NULL ? Checksum() : *recursiveChecksum);
 
-    SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] starting checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),checksum.getSum());
+    //SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] starting checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),checksum.getSum());
 
 	std::string mypath = path;
 	/** Stupid win32 is searching for all files without extension when *. is
@@ -429,28 +442,43 @@ int32 getFolderTreeContentsCheckSumRecursively(const string &path, const string
 
 	globfree(&globbuf);
 
-	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] ending checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),checksum.getSum());
+	//SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] ending checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),checksum.getSum());
 
 	if(recursiveChecksum != NULL) {
 		*recursiveChecksum = checksum;
 	}
 
-    return checksum.getSum();
+	crcTreeCache[cacheKey] = checksum.getFinalFileListSum();
+
+	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s] ending checksum = %d\n",__FILE__,__FUNCTION__,path.c_str(),crcTreeCache[cacheKey]);
+
+    return crcTreeCache[cacheKey];
 }
 
 vector<std::pair<string,int32> > getFolderTreeContentsCheckSumListRecursively(vector<string> paths, string pathSearchString, string filterFileExt, vector<std::pair<string,int32> > *recursiveMap) {
+	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__);
+
 	vector<std::pair<string,int32> > checksumFiles = (recursiveMap == NULL ? vector<std::pair<string,int32> >() : *recursiveMap);
-	size_t count = paths.size();
+	int count = paths.size();
-	for(unsigned int idx = 0; idx < count; ++idx) {
+	for(int idx = 0; idx < count; ++idx) {
 		string path = paths[idx] + pathSearchString;
 		getFolderTreeContentsCheckSumListRecursively(path, filterFileExt, &checksumFiles);
 	}
+
+	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d]\n",__FILE__,__FUNCTION__,__LINE__);
+
 	return checksumFiles;
 }
 
 //finds all filenames like path and gets the checksum of each file
 vector<std::pair<string,int32> > getFolderTreeContentsCheckSumListRecursively(const string &path, const string &filterFileExt, vector<std::pair<string,int32> > *recursiveMap) {
 
+	string cacheKey = path + "_" + filterFileExt;
+	static std::map<string,vector<std::pair<string,int32> > > crcTreeCache;
+	if(crcTreeCache.find(cacheKey) != crcTreeCache.end()) {
+		return crcTreeCache[cacheKey];
+	}
+
     vector<std::pair<string,int32> > checksumFiles = (recursiveMap == NULL ? vector<std::pair<string,int32> >() : *recursiveMap);
 
     SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s]\n",__FILE__,__FUNCTION__,path.c_str());
@@ -532,7 +560,9 @@ vector<std::pair<string,int32> > getFolderTreeContentsCheckSumListRecursively(co
 
 	SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s] scanning [%s]\n",__FILE__,__FUNCTION__,path.c_str());
 
-    return checksumFiles;
+	crcTreeCache[cacheKey] = checksumFiles;
+
+    return crcTreeCache[cacheKey];
 }
 
 string extractDirectoryPathFromFile(string filename)

source/shared_lib/sources/util/checksum.cpp

@@ -1,7 +1,7 @@
 // ==============================================================
 //	This file is part of Glest Shared Library (www.glest.org)
 //
-//	Copyright (C) 2001-2008 Marti<EFBFBD>o Figueroa
+//	Copyright (C) 2001-2008 Martio Figueroa
 //
 //	You can redistribute this code and/or modify it under
 //	the terms of the GNU General Public License as published
@@ -25,20 +25,26 @@ namespace Shared{ namespace Util{
 //	class Checksum
 // =====================================================
 
-Checksum::Checksum(){
+std::map<string,int32> Checksum::fileListCache;
+
+Checksum::Checksum() {
 	sum= 0;
 	r= 55665;
 	c1= 52845;
 	c2= 22719;
 }
 
-void Checksum::addByte(int8 value){
+void Checksum::addByte(int8 value) {
 	int32 cipher= (value ^ (r >> 8));
 
 	r= (cipher + r) * c1 + c2;
 	sum+= cipher;
 }
 
+void Checksum::addSum(int32 value) {
+	sum+= value;
+}
+
 void Checksum::addString(const string &value){
 	for(int i= 0; i<value.size(); ++i){
 		addByte(value[i]);
@@ -52,7 +58,6 @@ void Checksum::addFile(const string &path){
 void Checksum::addFileToSum(const string &path){
 
 	FILE* file= fopen(path.c_str(), "rb");
-
 	if(file!=NULL){
 
 		addString(lastFile(path));
@@ -73,13 +78,28 @@ void Checksum::addFileToSum(const string &path){
 
 int32 Checksum::getSum() {
 	if(fileList.size() > 0) {
+		SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] fileList.size() = %d\n",__FILE__,__FUNCTION__,__LINE__,fileList.size());
+
+		Checksum newResult;
 		for(std::map<string,int32>::iterator iterMap = fileList.begin();
-			iterMap != fileList.end(); iterMap++)
+			iterMap != fileList.end(); iterMap++) {
-		{
+			if(Checksum::fileListCache.find(iterMap->first) == Checksum::fileListCache.end()) {
-			addFileToSum(iterMap->first);
+				Checksum fileResult;
+				fileResult.addFileToSum(iterMap->first);
+				Checksum::fileListCache[iterMap->first] = fileResult.getSum();
+
+				SystemFlags::OutputDebug(SystemFlags::debugSystem,"In [%s::%s Line: %d] iterMap->first [%s] has CRC [%d]\n",__FILE__,__FUNCTION__,__LINE__,iterMap->first.c_str(),Checksum::fileListCache[iterMap->first]);
 			}
+			newResult.addSum(Checksum::fileListCache[iterMap->first]);
+		}
+		return newResult.getSum();
 	}
 	return sum;
 }
 
+int32 Checksum::getFinalFileListSum() {
+	sum = 0;
+	return getSum();
+}
+
 }}//end namespace