Read stamps from stamps.json

... while retaining all the functionality of stamps.def.

Also fix stamp names encoding only 32 bits of the timestamp, migrate from stamps.def to stamps.json if the latter doesn't exist, delete both on migration to the shared data directory, rescan stamps at startup, and make rescanning a painless process in general by removing invalid entries and adding missing entires at the beginning of the list.

src/client/Client.cpp

@@ -11,6 +11,7 @@
 #include "common/Platform.h"
 #include "common/String.h"
 #include "graphics/Graphics.h"
+#include "prefs/Prefs.h"
 #include "lua/CommandInterface.h"
 #include "gui/preview/Comment.h"
 #include "Config.h"
@@ -20,9 +21,11 @@
 #include <map>
 #include <iostream>
 #include <iomanip>
-#include <ctime>
 #include <cstdio>
 #include <fstream>
+#include <chrono>
+#include <algorithm>
+#include <set>
 
 Client::Client():
 	messageOfTheDay("Fetching the message of the day..."),
@@ -50,6 +53,19 @@ Client::Client():
 	firstRun = !prefs.BackedByFile();
 }
 
+void Client::MigrateStampsDef()
+{
+	std::vector<char> data;
+	if (!Platform::ReadFile(data, ByteString::Build(STAMPS_DIR, PATH_SEP_CHAR, "stamps.def")))
+	{
+		return;
+	}
+	for (auto i = 0; i < int(data.size()); i += 10)
+	{
+		stampIDs.push_back(ByteString(&data[0] + i, &data[0] + i + 10));
+	}
+}
+
 void Client::Initialize()
 {
 	auto &prefs = GlobalPrefs::Ref();
@@ -59,19 +75,17 @@ void Client::Initialize()
 		Platform::UpdateFinish();
 	}
 
-	//Read stamps library
+	stamps = std::make_unique<Prefs>(ByteString::Build(STAMPS_DIR, PATH_SEP_CHAR, "stamps.json"));
-	std::ifstream stampsLib;
+	stampIDs = stamps->Get("MostRecentlyUsedFirst", std::vector<ByteString>{});
-	stampsLib.open(ByteString::Build(STAMPS_DIR, PATH_SEP_CHAR, "stamps.def"), std::ios::binary);
-	while (!stampsLib.eof())
 	{
-		char data[11];
+		Prefs::DeferWrite dw(*stamps);
-		memset(data, 0, 11);
+		if (!stamps->BackedByFile())
-		stampsLib.read(data, 10);
+		{
-		if(!data[0])
+			MigrateStampsDef();
-			break;
+			WriteStamps();
-		stampIDs.push_back(data);
+		}
+		RescanStamps();
 	}
-	stampsLib.close();
 
 	//Begin version check
 	versionCheckRequest = std::make_unique<http::Request>(ByteString::Build(SCHEME, SERVER, "/Startup.json"));
@@ -462,16 +476,23 @@ RequestStatus Client::UploadSave(SaveInfo & save)
 
 void Client::MoveStampToFront(ByteString stampID)
 {
-	for (std::list<ByteString>::iterator iterator = stampIDs.begin(), end = stampIDs.end(); iterator != end; ++iterator)
+	auto it = std::find(stampIDs.begin(), stampIDs.end(), stampID);
+	auto changed = false;
+	if (it == stampIDs.end())
 	{
-		if((*iterator) == stampID)
+		stampIDs.push_back(stampID);
+		it = stampIDs.end() - 1;
+		changed = true;
+	}
+	else if (it != stampIDs.begin())
 	{
-			stampIDs.erase(iterator);
+		changed = true;
-			break;
 	}
+	if (changed)
+	{
+		std::rotate(stampIDs.begin(), it, it + 1);
+		WriteStamps();
 	}
-	stampIDs.push_front(stampID);
-	updateStamps();
 }
 
 SaveFile * Client::GetStamp(ByteString stampID)
@@ -487,32 +508,37 @@ SaveFile * Client::GetStamp(ByteString stampID)
 
 void Client::DeleteStamp(ByteString stampID)
 {
-	for (std::list<ByteString>::iterator iterator = stampIDs.begin(), end = stampIDs.end(); iterator != end; ++iterator)
+	auto it = std::remove(stampIDs.begin(), stampIDs.end(), stampID);
+	if (it != stampIDs.end())
 	{
-		if ((*iterator) == stampID)
+		stampIDs.erase(it, stampIDs.end());
-		{
+		WriteStamps();
-			ByteString stampFilename = ByteString::Build(STAMPS_DIR, PATH_SEP_CHAR, stampID, ".stm");
-			remove(stampFilename.c_str());
-			stampIDs.erase(iterator);
-			break;
 	}
-	}
-
-	updateStamps();
 }
 
 ByteString Client::AddStamp(GameSave * saveData)
 {
-	unsigned t=(unsigned)time(NULL);
+	auto now = (uint64_t)time(NULL);
-	if (lastStampTime!=t)
+	if (lastStampTime != now)
 	{
-		lastStampTime=t;
+		lastStampTime = now;
-		lastStampName=0;
+		lastStampName = 0;
 	}
 	else
-		lastStampName++;
+	{
-	ByteString saveID = ByteString::Build(Format::Hex(Format::Width(lastStampTime, 8)), Format::Hex(Format::Width(lastStampName, 2)));
+		lastStampName += 1;
-	ByteString filename = ByteString::Build(STAMPS_DIR, PATH_SEP_CHAR, saveID, ".stm");
+	}
+	ByteString saveID, filename;
+	while (true)
+	{
+		saveID = ByteString::Build(Format::Hex(Format::Width(lastStampTime, 8)), Format::Hex(Format::Width(lastStampName, 2)));
+		filename = ByteString::Build(STAMPS_DIR, PATH_SEP_CHAR, saveID, ".stm");
+		if (!Platform::FileExists(filename))
+		{
+			break;
+		}
+		lastStampName += 1;
+	}
 
 	Platform::MakeDirectory(STAMPS_DIR);
 
@@ -520,7 +546,7 @@ ByteString Client::AddStamp(GameSave * saveData)
 	stampInfo["type"] = "stamp";
 	stampInfo["username"] = authUser.Username;
 	stampInfo["name"] = filename;
-	stampInfo["date"] = (Json::Value::UInt64)time(NULL);
+	stampInfo["date"] = Json::Value::UInt64(now);
 	if (authors.size() != 0)
 	{
 		// This is a stamp, always append full authorship info (even if same user)
@@ -534,66 +560,61 @@ ByteString Client::AddStamp(GameSave * saveData)
 		return "";
 
 	Platform::WriteFile(gameData, filename);
-
+	MoveStampToFront(saveID);
-	stampIDs.push_front(saveID);
-
-	updateStamps();
-
 	return saveID;
 }
 
-void Client::updateStamps()
-{
-	Platform::MakeDirectory(STAMPS_DIR);
-
-	std::ofstream stampsStream;
-	stampsStream.open(ByteString::Build(STAMPS_DIR, PATH_SEP_CHAR, "stamps.def").c_str(), std::ios::binary);
-	for (std::list<ByteString>::const_iterator iterator = stampIDs.begin(), end = stampIDs.end(); iterator != end; ++iterator)
-	{
-		stampsStream.write((*iterator).c_str(), 10);
-	}
-	stampsStream.write("\0", 1);
-	stampsStream.close();
-	return;
-}
-
 void Client::RescanStamps()
 {
-	stampIDs.clear();
+	ByteString extension = ".stm";
-	for (auto &stamp : Platform::DirectorySearch("stamps", "", { ".stm" }))
+	std::set<ByteString> stampFilesSet;
+	for (auto &stampID : Platform::DirectorySearch("stamps", "", { extension }))
 	{
-		if (stamp.size() == 14)
+		stampFilesSet.insert(stampID.substr(0, stampID.size() - extension.size()));
+	}
+	std::vector<ByteString> newStampIDs;
+	auto changed = false;
+	for (auto &stampID : stampIDs)
 	{
-			stampIDs.push_front(stamp.Substr(0, 10));
+		if (stampFilesSet.find(stampID) == stampFilesSet.end())
+		{
+			changed = true;
+		}
+		else
+		{
+			newStampIDs.push_back(stampID);
 		}
 	}
-	stampIDs.sort(std::greater<ByteString>());
+	auto oldCount = newStampIDs.size();
-	updateStamps();
+	auto stampIDsSet = std::set<ByteString>(stampIDs.begin(), stampIDs.end());
+	for (auto &stampID : stampFilesSet)
+	{
+		if (stampIDsSet.find(stampID) == stampIDsSet.end())
+		{
+			newStampIDs.push_back(stampID);
+			changed = true;
+		}
+	}
+	if (changed)
+	{
+		// Move newly discovered stamps to front.
+		std::rotate(newStampIDs.begin(), newStampIDs.begin() + oldCount, newStampIDs.end());
+		stampIDs = newStampIDs;
+		WriteStamps();
+	}
 }
 
-int Client::GetStampsCount()
+void Client::WriteStamps()
 {
-	return stampIDs.size();
+	if (stampIDs.size())
+	{
+		stamps->Set("MostRecentlyUsedFirst", stampIDs);
+	}
 }
 
-std::vector<ByteString> Client::GetStamps(int start, int count)
+const std::vector<ByteString> &Client::GetStamps() const
 {
-	int size = (int)stampIDs.size();
+	return stampIDs;
-	if (start+count > size)
-	{
-		if(start > size)
-			return std::vector<ByteString>();
-		count = size-start;
-	}
-
-	std::vector<ByteString> stampRange;
-	int index = 0;
-	for (std::list<ByteString>::const_iterator iterator = stampIDs.begin(), end = stampIDs.end(); iterator != end; ++iterator, ++index)
-	{
-		if(index>=start && index < start+count)
-			stampRange.push_back(*iterator);
-	}
-	return stampRange;
 }
 
 RequestStatus Client::ExecVote(int saveID, int direction)
@@ -1209,6 +1230,7 @@ String Client::DoMigration(ByteString fromDir, ByteString toDir)
 
 	// Do actual migration
 	Platform::RemoveFile(fromDir + "stamps/stamps.def");
+	Platform::RemoveFile(fromDir + "stamps/stamps.json");
 	migrateList(stamps, "stamps", "Stamps");
 	migrateList(saves, "Saves", "Saves");
 	if (!scripts.empty())
@@ -1238,7 +1260,6 @@ String Client::DoMigration(ByteString fromDir, ByteString toDir)
 	// chdir into the new directory
 	Platform::ChangeDir(toDir);
 
-	if (scripts.size())
 	RescanStamps();
 
 	logFile << std::endl << std::endl << "Migration complete. Results: " << result.Build().ToUtf8();

src/client/Client.h

@@ -3,6 +3,7 @@
 #include "common/ExplicitSingleton.h"
 #include "User.h"
 #include <vector>
+#include <cstdint>
 #include <list>
 #include <memory>
 #include <json/json.h>
@@ -37,6 +38,7 @@ public:
 	UpdateInfo(int time, ByteString file, String changelog, BuildType type) : File(file), Changelog(changelog), Major(0), Minor(0), Build(0), Time(time), Type(type) {}
 };
 
+class Prefs;
 class RequestListener;
 class ClientListener;
 namespace http
@@ -57,9 +59,9 @@ private:
 	String lastError;
 	bool firstRun;
 
-	std::list<ByteString> stampIDs;
+	std::vector<ByteString> stampIDs;
-	unsigned lastStampTime;
+	uint64_t lastStampTime = 0;
-	int lastStampName;
+	int lastStampName = 0;
 
 	//Auth session
 	User authUser;
@@ -72,6 +74,10 @@ private:
 	// Save stealing info
 	Json::Value authors;
 
+	std::unique_ptr<Prefs> stamps;
+	void MigrateStampsDef();
+	void WriteStamps();
+
 public:
 
 	std::vector<ClientListener*> listeners;
@@ -111,12 +117,9 @@ public:
 	SaveFile * GetStamp(ByteString stampID);
 	void DeleteStamp(ByteString stampID);
 	ByteString AddStamp(GameSave * saveData);
-	std::vector<ByteString> GetStamps(int start, int count);
 	void RescanStamps();
-	int GetStampsCount();
+	const std::vector<ByteString> &GetStamps() const;
-	SaveFile * GetFirstStamp();
 	void MoveStampToFront(ByteString stampID);
-	void updateStamps();
 
 	RequestStatus AddComment(int saveID, String comment);
 

src/gui/game/GameView.cpp

@@ -1553,10 +1553,10 @@ void GameView::OnKeyPress(int key, int scan, bool repeat, bool shift, bool ctrl,
 		break;
 	case SDL_SCANCODE_L:
 	{
-		std::vector<ByteString> stampList = Client::Ref().GetStamps(0, 1);
+		auto &stampIDs = Client::Ref().GetStamps();
-		if (stampList.size())
+		if (stampIDs.size())
 		{
-			SaveFile *saveFile = Client::Ref().GetStamp(stampList[0]);
+			SaveFile *saveFile = Client::Ref().GetStamp(stampIDs[0]);
 			if (!saveFile || !saveFile->GetGameSave())
 				break;
 			c->LoadStamp(saveFile->GetGameSave());

src/gui/localbrowser/LocalBrowserController.cpp

@@ -65,7 +65,6 @@ void LocalBrowserController::removeSelectedC()
 		}
 		void after() override
 		{
-			Client::Ref().updateStamps();
 			c->RefreshSavesList();
 		}
 	};
@@ -75,11 +74,6 @@ void LocalBrowserController::removeSelectedC()
 }
 
 void LocalBrowserController::RescanStamps()
-{
-	new ConfirmPrompt("Rescan", "Rescanning the stamps folder can find stamps added to the stamps folder or recover stamps when the stamps.def file has been lost or damaged. However, be warned that this will mess up the current sorting order", { [this] { rescanStampsC(); } });
-}
-
-void LocalBrowserController::rescanStampsC()
 {
 	browserModel->RescanStamps();
 	browserModel->UpdateSavesList(browserModel->GetPageNum());

src/gui/localbrowser/LocalBrowserController.h

@@ -19,7 +19,6 @@ public:
 	void ClearSelection();
 	void Selected(ByteString stampID, bool selected);
 	void RescanStamps();
-	void rescanStampsC();
 	void RefreshSavesList();
 	void OpenSave(SaveFile * stamp);
 	bool GetMoveToFront();

src/gui/localbrowser/LocalBrowserModel.cpp

@@ -1,21 +1,18 @@
 #include "LocalBrowserModel.h"
-
 #include "LocalBrowserView.h"
-
-#include <cmath>
-
 #include "client/Client.h"
 #include "client/SaveFile.h"
-
 #include "common/tpt-minmax.h"
+#include <algorithm>
+
+constexpr auto pageSize = 20;
 
 LocalBrowserModel::LocalBrowserModel():
 	stamp(NULL),
 	currentPage(1),
 	stampToFront(1)
 {
-	//stampIDs = Client::Ref().GetStamps();
+	stampIDs = Client::Ref().GetStamps();
-	stampIDs = Client::Ref().GetStamps(0, 16);
 }
 
 
@@ -82,9 +79,9 @@ void LocalBrowserModel::UpdateSavesList(int pageNumber)
 		delete tempSavesList[i];
 	}*/
 
-	stampIDs = Client::Ref().GetStamps((pageNumber-1)*20, 20);
+	stampIDs = Client::Ref().GetStamps();
-
+	auto size = int(stampIDs.size());
-	for (size_t i = 0; i < stampIDs.size(); i++)
+	for (int i = (currentPage - 1) * pageSize; i < size && i < currentPage * pageSize; i++)
 	{
 		SaveFile * tempSave = Client::Ref().GetStamp(stampIDs[i]);
 		if (tempSave)
@@ -102,37 +99,28 @@ void LocalBrowserModel::RescanStamps()
 
 int LocalBrowserModel::GetPageCount()
 {
-	return std::max(1, (int)(std::ceil(float(Client::Ref().GetStampsCount())/20.0f)));
+	auto size = int(stampIDs.size());
+	return size / pageSize + ((size % pageSize) ? 1 : 0);
 }
 
 void LocalBrowserModel::SelectSave(ByteString stampID)
 {
-	for (size_t i = 0; i < selected.size(); i++)
+	if (std::find(selected.begin(), selected.end(), stampID) != selected.end())
-	{
-		if (selected[i] == stampID)
 	{
 		return;
 	}
-	}
 	selected.push_back(stampID);
 	notifySelectedChanged();
 }
 
 void LocalBrowserModel::DeselectSave(ByteString stampID)
 {
-	bool changed = false;
+	auto it = std::remove(selected.begin(), selected.end(), stampID);
-restart:
+	if (it != selected.end())
-	for (size_t i = 0; i < selected.size(); i++)
 	{
-		if (selected[i] == stampID)
+		selected.erase(it, selected.end());
-		{
-			selected.erase(selected.begin()+i);
-			changed = true;
-			goto restart; //Just ensure all cases are removed.
-		}
-	}
-	if(changed)
 		notifySelectedChanged();
+	}
 }
 
 void LocalBrowserModel::notifySelectedChanged()

src/lua/LuaScriptInterface.cpp

@@ -1896,18 +1896,19 @@ int LuaScriptInterface::simulation_loadStamp(lua_State * l)
 	SaveFile * tempfile = NULL;
 	int x = luaL_optint(l,2,0);
 	int y = luaL_optint(l,3,0);
+	auto &client = Client::Ref();
 	if (lua_isstring(l, 1)) //Load from 10 char name, or full filename
 	{
 		auto filename = tpt_lua_optByteString(l, 1, "");
-		tempfile = Client::Ref().GetStamp(filename);
+		tempfile = client.GetStamp(filename);
 	}
 	if ((!tempfile || !tempfile->GetGameSave()) && lua_isnumber(l, 1)) //Load from stamp ID
 	{
 		i = luaL_optint(l, 1, 0);
-		int stampCount = Client::Ref().GetStampsCount();
+		auto &stampIDs = client.GetStamps();
-		if (i < 0 || i >= stampCount)
+		if (i < 0 || i >= int(stampIDs.size()))
 			return luaL_error(l, "Invalid stamp ID: %d", i);
-		tempfile = Client::Ref().GetStamp(Client::Ref().GetStamps(0, stampCount)[i]);
+		tempfile = client.GetStamp(stampIDs[i]);
 	}
 
 	if (tempfile)
@@ -1920,7 +1921,7 @@ int LuaScriptInterface::simulation_loadStamp(lua_State * l)
 			if (tempfile->GetGameSave()->authors.size())
 			{
 				tempfile->GetGameSave()->authors["type"] = "luastamp";
-				Client::Ref().MergeStampAuthorInfo(tempfile->GetGameSave()->authors);
+				client.MergeStampAuthorInfo(tempfile->GetGameSave()->authors);
 			}
 		}
 		else
@@ -1942,17 +1943,17 @@ int LuaScriptInterface::simulation_loadStamp(lua_State * l)
 
 int LuaScriptInterface::simulation_deleteStamp(lua_State * l)
 {
-	int stampCount = Client::Ref().GetStampsCount();
+	auto &client = Client::Ref();
-	std::vector<ByteString> stamps = Client::Ref().GetStamps(0, stampCount);
+	auto &stampIDs = client.GetStamps();
 
 	if (lua_isstring(l, 1)) //note: lua_isstring returns true on numbers too
 	{
 		auto filename = tpt_lua_optByteString(l, 1, "");
-		for (auto &stamp : stamps)
+		for (auto &stampID : stampIDs)
 		{
-			if (stamp == filename)
+			if (stampID == filename)
 			{
-				Client::Ref().DeleteStamp(stamp);
+				client.DeleteStamp(stampID);
 				return 0;
 			}
 		}
@@ -1960,9 +1961,9 @@ int LuaScriptInterface::simulation_deleteStamp(lua_State * l)
 	if (lua_isnumber(l, 1)) //Load from stamp ID
 	{
 		int i = luaL_optint(l, 1, 0);
-		if (i < 0 || i >= stampCount)
+		if (i < 0 || i >= int(stampIDs.size()))
 			return luaL_error(l, "Invalid stamp ID: %d", i);
-		Client::Ref().DeleteStamp(stamps[i]);
+		client.DeleteStamp(stampIDs[i]);
 		return 0;
 	}
 	lua_pushnumber(l, -1);