Compress font data

data/font.h

@@ -1,11 +0,0 @@
-#pragma once
-#define FONT_H 12
-#ifndef FONTEDITOR
-extern const unsigned char font_data[];
-extern const unsigned int font_ptrs[];
-extern const unsigned int font_ranges[][2];
-#else
-extern unsigned char *font_data;
-extern unsigned int *font_ptrs;
-extern unsigned int (*font_ranges)[2];
-#endif

data/meson.build

@@ -1,9 +1,15 @@
+to_array = generator(
+	import('python').find_installation('python3'),
+	output: [ '@BASENAME@.cpp', '@BASENAME@.h' ],
+	arguments: [ join_paths(meson.current_source_dir(), 'to_array.py'), '@BUILD_DIR@', '@OUTPUT0@', '@OUTPUT1@', '@INPUT@', '@EXTRA_ARGS@' ]
+)
+
 data_files = files(
-	'font.cpp',
 	'hmap.cpp',
 	'icon.cpp',
 	'images.cpp',
 )
+data_files += to_array.process('font.bz2', extra_args: 'compressed_font_data')
 
 powder_files += data_files
 render_files += data_files

data/to_array.py

@@ -0,0 +1,16 @@
+import sys
+
+build_dir = sys.argv[1]
+output_cpp = sys.argv[2]
+output_h = sys.argv[3]
+input_any = sys.argv[4]
+symbol_name = sys.argv[5]
+
+with open(input_any, 'rb') as input_any_f:
+	data = input_any_f.read()
+
+with open(output_cpp, 'w') as output_cpp_f:
+	output_cpp_f.write('#include "{0}"\nconst unsigned char {1}[] = {{ {2} }}; const unsigned int {1}_size = {3};\n'.format(output_h, symbol_name, ','.join([ str(b) for b in data ]), len(data)))
+
+with open(output_h, 'w') as output_h_f:
+	output_h_f.write('#pragma once\nextern const unsigned char {0}[]; extern const unsigned int {0}_size;\n'.format(symbol_name))

fonttool.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 
+import bz2
 import math
 import re
 import argparse
 
 CP_MAX = 0x10FFFF
-FONT_CPP = "data/font.cpp"
+FONT_CPP = "data/font.bz2"
 FONT_HEIGHT = 12
-PTRS_PER_LINE = 8
 
 
 class ReadBDFError(RuntimeError):
@@ -17,57 +17,27 @@ class ReadBDFError(RuntimeError):
 
 class FontTool:
     def __init__(self):
-        with open(FONT_CPP) as font_cpp:
+        with open(FONT_CPP, 'rb') as font_cpp:
-            self.font_cpp_data = font_cpp.read()
+            font_cpp_data = bz2.decompress(font_cpp.read())
-        font_data = ([int(s, 16) for s in re.findall(r'\w+', re.search(r'font_data[^{]*{([^;]+);', self.font_cpp_data,
+        i = 0
-                                                                       re.MULTILINE | re.DOTALL)[1])])
-        font_ptrs = ([int(s, 16) for s in re.findall(r'\w+', re.search(r'font_ptrs[^{]*{([^;]+);', self.font_cpp_data,
-                                                                       re.MULTILINE | re.DOTALL)[1])])
-        font_ranges = ([int(s, 16) for s in re.findall(r'\w+',
-                                                       re.search(r'font_ranges[^{]*{([^;]+);', self.font_cpp_data,
-                                                                 re.MULTILINE | re.DOTALL)[1])])
         self.code_points = [False for _ in range(CP_MAX + 2)]
-        ptrs_ptr = 0
+        while i < len(font_cpp_data):
-        for i in range(len(font_ranges) // 2 - 1):
+            cp = font_cpp_data[i] | (font_cpp_data[i + 1] << 8) | (font_cpp_data[i + 2] << 16)
-            for cp in range(font_ranges[i * 2], font_ranges[i * 2 + 1] + 1):
+            width = font_cpp_data[i + 3]
-                base = font_ptrs[ptrs_ptr]
+            n = i + 4 + 3 * width
-                self.code_points[cp] = font_data[base: (base + math.ceil(font_data[base] * FONT_HEIGHT / 4) + 1)]
+            self.code_points[cp] = font_cpp_data[(i + 3): n]
-                ptrs_ptr += 1
+            i = n
 
     def commit(self):
-        new_ranges = []
+        l = []
-        in_range = False
         for i, data in enumerate(self.code_points):
-            if in_range and not data:
+            if data:
-                new_ranges[-1].append(i - 1)
+                l.append(i & 0xFF)
-                in_range = False
+                l.append((i >> 8) & 0xFF)
-            elif not in_range and data:
+                l.append((i >> 16) & 0xFF)
-                in_range = True
+                l += data
-                new_ranges.append([i])
+        with open(FONT_CPP, 'wb') as font_cpp:
-        font_data_lines_hex = [['0x%02X' % v for v in d] for d in filter(lambda x: x, self.code_points)]
+            font_cpp.write(bz2.compress(bytes(l)))
-        font_data_lines = [len(h) > 1 and h[0] + ',   ' + ', '.join(h[1:]) + ',' or '0x00,  ' for h in
-                           font_data_lines_hex]
-        font_cpp_data = re.sub(r'font_data[^{]*{([^;]+);',
-                               'font_data[] = {\n    ' + '\n    '.join(font_data_lines) + '\n};', self.font_cpp_data)
-        font_ptrs_blocks = []
-        data_ptr = 0
-        for ran in new_ranges:
-            block = []
-            for cp in range(ran[0], ran[1] + 1):
-                block.append(data_ptr)
-                data_ptr += math.ceil(self.code_points[cp][0] * FONT_HEIGHT / 4) + 1
-            font_ptrs_wrapped = []
-            for i in range(0, len(block), PTRS_PER_LINE):
-                font_ptrs_wrapped.append(', '.join(['0x%08X' % v for v in block[i: (i + PTRS_PER_LINE)]]))
-            font_ptrs_blocks.append(',\n    '.join(font_ptrs_wrapped))
-        font_cpp_data = re.sub(r'font_ptrs[^{]*{([^;]+);',
-                               'font_ptrs[] = {\n    ' + ',\n\n    '.join(font_ptrs_blocks) + ',\n};', font_cpp_data)
-        font_ranges_lines = ['{ 0x%06X, 0x%06X },' % (r[0], r[1]) for r in new_ranges]
-        font_cpp_data = re.sub(r'font_ranges[^{]*{([^;]+);',
-                               'font_ranges[][2] = {\n    ' + '\n    '.join(font_ranges_lines) + '\n    { 0, 0 },\n};',
-                               font_cpp_data)
-        with open(FONT_CPP, 'w') as font_cpp:
-            font_cpp.write(font_cpp_data)
 
     def pack(cp_matrix):
         width = 0

meson.build

@@ -4,6 +4,7 @@ project('the-powder-toy', [ 'c', 'cpp' ], version: 'the.cake.is.a.lie', default_
 	'backend_startup_project=powder',
 ])
 
+prog_python3 = import('python').find_installation('python3')
 cpp_compiler = meson.get_compiler('cpp')
 
 project_c_args = []

src/common/bz2wrap.cpp

@@ -0,0 +1,106 @@
+#include "bz2wrap.h"
+
+#include <bzlib.h>
+
+#include <memory>
+#include <functional>
+#include <vector>
+#include <algorithm>
+
+static size_t outputSizeIncrement = 0x100000U;
+
+BZ2WCompressResult BZ2WCompress(std::vector<char> &dest, const char *srcData, size_t srcSize, size_t maxSize)
+{
+	bz_stream stream;
+	stream.bzalloc = NULL;
+	stream.bzfree = NULL;
+	stream.opaque = NULL;
+	if (BZ2_bzCompressInit(&stream, 9, 0, 0) != BZ_OK)
+	{
+		return BZ2WCompressNomem;
+	}
+	std::unique_ptr<bz_stream, std::function<int (bz_stream *)>> bz2Data(&stream, BZ2_bzCompressEnd);
+	stream.next_in = const_cast<char *>(srcData); // I hope bz2 doesn't actually write anything here...
+	stream.avail_in = srcSize;
+	dest.resize(0);
+	bool done = false;
+	while (!done)
+	{
+		size_t oldSize = dest.size();
+		size_t newSize = oldSize + outputSizeIncrement;
+		if (maxSize && newSize > maxSize)
+		{
+			newSize = maxSize;
+		}
+		if (oldSize == newSize)
+		{
+			return BZ2WCompressLimit;
+		}
+		dest.resize(newSize);
+		stream.next_out = &dest[stream.total_out_lo32];
+		stream.avail_out = dest.size() - stream.total_out_lo32;
+		if (BZ2_bzCompress(&stream, BZ_FINISH) == BZ_STREAM_END)
+		{
+			done = true;
+		}
+	}
+	dest.resize(stream.total_out_lo32);
+	return BZ2WCompressOk;
+}
+
+BZ2WDecompressResult BZ2WDecompress(std::vector<char> &dest, const char *srcData, size_t srcSize, size_t maxSize)
+{
+	bz_stream stream;
+	stream.bzalloc = NULL;
+	stream.bzfree = NULL;
+	stream.opaque = NULL;
+	if (BZ2_bzDecompressInit(&stream, 0, 0) != BZ_OK)
+	{
+		return BZ2WDecompressNomem;
+	}
+	std::unique_ptr<bz_stream, std::function<int (bz_stream *)>> bz2Data(&stream, BZ2_bzDecompressEnd); 
+	stream.next_in = const_cast<char *>(srcData); // I hope bz2 doesn't actually write anything here...
+	stream.avail_in = srcSize;
+	dest.resize(0);
+	bool done = false;
+	while (!done)
+	{
+		size_t oldSize = dest.size();
+		size_t newSize = oldSize + outputSizeIncrement;
+		if (maxSize && newSize > maxSize)
+		{
+			newSize = maxSize;
+		}
+		if (oldSize == newSize)
+		{
+			return BZ2WDecompressLimit;
+		}
+		dest.resize(newSize);
+		stream.next_out = &dest[stream.total_out_lo32];
+		stream.avail_out = dest.size() - stream.total_out_lo32;
+		switch (BZ2_bzDecompress(&stream))
+		{
+		case BZ_OK:
+			if (!stream.avail_in && stream.avail_out)
+			{
+				return BZ2WDecompressEof;
+			}
+			break;
+
+		case BZ_MEM_ERROR:
+			return BZ2WDecompressNomem;
+
+		case BZ_DATA_ERROR:
+			return BZ2WDecompressBad;
+
+		case BZ_DATA_ERROR_MAGIC:
+			return BZ2WDecompressType;
+
+		case BZ_STREAM_END:
+			done = true;
+			break;
+		}
+	}
+	dest.resize(stream.total_out_lo32);
+	return BZ2WDecompressOk;
+}

src/common/bz2wrap.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <vector>
+
+enum BZ2WCompressResult
+{
+	BZ2WCompressOk,
+	BZ2WCompressNomem,
+	BZ2WCompressLimit,
+};
+BZ2WCompressResult BZ2WCompress(std::vector<char> &dest, const char *srcData, size_t srcSize, size_t maxSize = 0);
+
+enum BZ2WDecompressResult
+{
+	BZ2WDecompressOk,
+	BZ2WDecompressNomem,
+	BZ2WDecompressLimit,
+	BZ2WDecompressType,
+	BZ2WDecompressBad,
+	BZ2WDecompressEof,
+};
+BZ2WDecompressResult BZ2WDecompress(std::vector<char> &dest, const char *srcData, size_t srcSize, size_t maxSize = 0);

src/common/meson.build

@@ -1,4 +1,5 @@
 common_files += files(
+	'bz2wrap.cpp',
 	'String.cpp',
 	'tpt-rand.cpp',
 )

src/graphics/FontReader.cpp

@@ -1,5 +1,12 @@
 #include "FontReader.h"
 
+#include "common/bz2wrap.h"
+#include "font.h"
+
+unsigned char *font_data = nullptr;
+unsigned int *font_ptrs = nullptr;
+unsigned int (*font_ranges)[2] = nullptr;
+
 FontReader::FontReader(unsigned char const *_pointer):
 	pointer(_pointer + 1),
 	width(*_pointer),
@@ -8,8 +15,78 @@ FontReader::FontReader(unsigned char const *_pointer):
 {
 }
 
+static bool InitFontData()
+{
+	static std::vector<char> fontDataBuf;
+	static std::vector<int> fontPtrsBuf;
+	static std::vector< std::array<int, 2> > fontRangesBuf;
+	if (BZ2WDecompress(fontDataBuf, reinterpret_cast<const char *>(compressed_font_data), compressed_font_data_size) != BZ2WDecompressOk)
+	{
+		return false;
+	}
+	int first = -1;
+	int last = -1;
+	char *begin = &fontDataBuf[0];
+	char *ptr = &fontDataBuf[0];
+	char *end = &fontDataBuf[0] + fontDataBuf.size();
+	while (ptr != end)
+	{
+		if (ptr + 4 > end)
+		{
+			return false;
+		}
+		auto codePoint = *reinterpret_cast<uint32_t *>(ptr) & 0xFFFFFFU;
+		if (codePoint >= 0x110000U)
+		{
+			return false;
+		}
+		auto width = *reinterpret_cast<uint8_t *>(ptr + 3);
+		if (width > 64)
+		{
+			return false;
+		}
+		if (ptr + 4 + width * 3 > end)
+		{
+			return false;
+		}
+		auto cp = (int)codePoint;
+		if (last >= cp)
+		{
+			return false;
+		}
+		if (first != -1 && last + 1 < cp)
+		{
+			fontRangesBuf.push_back({ { first, last } });
+			first = -1;
+		}
+		if (first == -1)
+		{
+			first = cp;
+		}
+		last = cp;
+		fontPtrsBuf.push_back(ptr + 3 - begin);
+		ptr += width * 3 + 4;
+	}
+	if (first != -1)
+	{
+		fontRangesBuf.push_back({ { first, last } });
+	}
+	fontRangesBuf.push_back({ { 0, 0 } });
+	font_data = reinterpret_cast<unsigned char *>(fontDataBuf.data());
+	font_ptrs = reinterpret_cast<unsigned int *>(fontPtrsBuf.data());
+	font_ranges = reinterpret_cast<unsigned int (*)[2]>(fontRangesBuf.data());
+	return true;
+}
+
 unsigned char const *FontReader::lookupChar(String::value_type ch)
 {
+	if (!font_data)
+	{
+		if (!InitFontData())
+		{
+			throw std::runtime_error("font data corrupt");
+		}
+	}
 	size_t offset = 0;
 	for(int i = 0; font_ranges[i][1]; i++)
 		if(font_ranges[i][0] > ch)

src/graphics/FontReader.h

@@ -2,7 +2,8 @@
 #include <cstddef>
 
 #include "common/String.h"
-#include "font.h"
+
+#define FONT_H 12
 
 class FontReader
 {

src/gui/font/FontEditor.cpp

@@ -5,6 +5,7 @@
 #include <iostream>
 
 #include "FontEditor.h"
+#include "common/bz2wrap.h"
 
 #include "Config.h"
 #include "gui/interface/Textbox.h"
@@ -16,153 +17,128 @@
 #include "gui/interface/ScrollPanel.h"
 #include "graphics/Graphics.h"
 
-unsigned char *font_data;
+extern unsigned char *font_data;
-unsigned int *font_ptrs;
+extern unsigned int *font_ptrs;
-unsigned int (*font_ranges)[2];
+extern unsigned int (*font_ranges)[2];
 
 void FontEditor::ReadDataFile(ByteString dataFile)
 {
 	std::fstream file;
-	file.open(dataFile, std::ios_base::in);
+	file.open(dataFile, std::ios_base::in | std::ios_base::binary);
 	if(!file)
 		throw std::runtime_error("Could not open " + dataFile);
-	file >> std::skipws;
+	file.seekg(0, std::ios_base::end);
+	std::vector<char> fileData(file.tellg());
+	file.seekg(0);
+	file.read(&fileData[0], fileData.size());
+	file.close();
 
-	ByteString word;
+	std::vector<char> fontDataBuf;
-
+	std::vector<int> fontPtrsBuf;
-	while(word != "font_data[]")
+	std::vector< std::array<int, 2> > fontRangesBuf;
-		file >> word;
+	if (BZ2WDecompress(fontDataBuf, fileData.data(), fileData.size()) != BZ2WDecompressOk)
-	file >> word >> word;
+	{
-
+		throw std::runtime_error("Could not decompress font data");
-	size_t startFontData = file.tellg();
+	}
+	int first = -1;
+	int last = -1;
+	char *begin = &fontDataBuf[0];
+	char *ptr = &fontDataBuf[0];
+	char *end = &fontDataBuf[0] + fontDataBuf.size();
+	while (ptr != end)
+	{
+		if (ptr + 4 > end)
+		{
+			throw std::runtime_error("Could not decompress font data");
+		}
+		auto codePoint = *reinterpret_cast<uint32_t *>(ptr) & 0xFFFFFFU;
+		if (codePoint >= 0x110000U)
+		{
+			throw std::runtime_error("Could not decompress font data");
+		}
+		auto width = *reinterpret_cast<uint8_t *>(ptr + 3);
+		if (width > 64)
+		{
+			throw std::runtime_error("Could not decompress font data");
+		}
+		if (ptr + 4 + width * 3 > end)
+		{
+			throw std::runtime_error("Could not decompress font data");
+		}
+		auto cp = (int)codePoint;
+		if (last >= cp)
+		{
+			throw std::runtime_error("Could not decompress font data");
+		}
+		if (first != -1 && last + 1 < cp)
+		{
+			fontRangesBuf.push_back({ { first, last } });
+			first = -1;
+		}
+		if (first == -1)
+		{
+			first = cp;
+		}
+		last = cp;
+		fontPtrsBuf.push_back(ptr + 3 - begin);
+		ptr += width * 3 + 4;
+	}
+	if (first != -1)
+	{
+		fontRangesBuf.push_back({ { first, last } });
+	}
+	fontRangesBuf.push_back({ { 0, 0 } });
 
 	fontData.clear();
-	do
+	for (auto ch : fontDataBuf)
 	{
-		unsigned int value;
+		fontData.push_back(ch);
-		file >> std::hex >> value;
-		if(!file.fail())
-		{
-			fontData.push_back(value);
-			file >> word;
-		}
 	}
-	while(!file.fail());
-	file.clear();
-
-	size_t endFontData = file.tellg();
-
-	while(word != "font_ptrs[]")
-		file >> word;
-	file >> word >> word;
-
-	size_t startFontPtrs = file.tellg();
-
 	fontPtrs.clear();
-	do
+	for (auto ptr : fontPtrsBuf)
 	{
-		unsigned int value;
+		fontPtrs.push_back(ptr);
-		file >> std::hex >> value;
-		if(!file.fail())
-		{
-			fontPtrs.push_back(value);
-			file >> word;
-		}
 	}
-	while(!file.fail());
-	file.clear();
-
-	size_t endFontPtrs = file.tellg();
-
-	while(word != "font_ranges[][2]")
-		file >> word;
-	file >> word >> word;
-
-	size_t startFontRanges = file.tellg();
-
 	fontRanges.clear();
-	while(true)
+	for (auto rng : fontRangesBuf)
 	{
-		unsigned int value1, value2;
+		fontRanges.push_back({ { (unsigned int)rng[0], (unsigned int)rng[1] } });
-		file >> word >> std::hex >> value1 >> word >> std::hex >> value2 >> word;
-		if(file.fail())
-			break;
-		fontRanges.push_back({value1, value2});
-		if(!value2)
-			break;
 	}
-	file.clear();
-
-	size_t endFontRanges = file.tellg();
-
-	do
-	{
-		file >> word;
-	}
-	while(!file.fail());
-	file.clear();
-	size_t eof = file.tellg();
-
-	file.seekg(0);
-	beforeFontData = ByteString(startFontData, 0);
-	file.read(&beforeFontData[0], startFontData);
-
-	file.seekg(endFontData);
-	afterFontData = ByteString(startFontPtrs - endFontData, 0);
-	file.read(&afterFontData[0], startFontPtrs - endFontData);
-
-	file.seekg(endFontPtrs);
-	afterFontPtrs = ByteString(startFontRanges - endFontPtrs, 0);
-	file.read(&afterFontPtrs[0], startFontRanges - endFontPtrs);
-
-	file.seekg(endFontRanges);
-	afterFontRanges = ByteString(eof - endFontRanges, 0);
-	file.read(&afterFontRanges[0], eof - endFontRanges);
-	file.close();
 }
 
 void FontEditor::WriteDataFile(ByteString dataFile, std::vector<unsigned char> const &fontData, std::vector<unsigned int> const &fontPtrs, std::vector<std::array<unsigned int, 2> > const &fontRanges)
 {
 	std::fstream file;
-	file.open(dataFile, std::ios_base::out | std::ios_base::trunc);
+	file.open(dataFile, std::ios_base::out | std::ios_base::trunc | std::ios_base::binary);
 	if(!file)
 		throw std::runtime_error("Could not open " + dataFile);
 
-	file << std::setfill('0') << std::hex << std::uppercase;
+	std::vector<char> uncompressed;
-	file << beforeFontData << std::endl;
-
 	size_t pos = 0;
-	size_t ptrpos = 0;
+	for (size_t i = 0; pos < fontPtrs.size() && fontRanges[i][1]; i++)
-	while(pos < fontData.size())
 	{
-		file << "    " << "0x" << std::setw(2) << (unsigned int)fontData[pos] << ",  ";
+		for (String::value_type ch = fontRanges[i][0]; ch <= fontRanges[i][1]; ch++)
-		for(pos++; pos < fontData.size() && (ptrpos == fontPtrs.size() - 1 || pos < (size_t)fontPtrs[ptrpos + 1]); pos++)
-			file << " " << "0x" << std::setw(2) << (unsigned int)fontData[pos] << ",";
-		file << std::endl;
-		ptrpos++;
-	}
-	file << afterFontData;
-
-	pos = 0;
-	for(size_t i = 0; pos < fontPtrs.size() && fontRanges[i][1]; i++)
-	{
-		bool first = true;
-		for(String::value_type ch = fontRanges[i][0]; ch <= fontRanges[i][1]; ch++)
 		{
-			if(!(ch & 0x7) || first)
+			uncompressed.push_back((char)ch);
-				file << std::endl << "    ";
+			uncompressed.push_back((char)(ch >> 8));
-			else
+			uncompressed.push_back((char)(ch >> 16));
-				file << " ";
+			auto ptr = fontPtrs[pos++];
-			first = false;
+			auto width = fontData[ptr];
-			file << "0x" << std::setw(8) << (unsigned int)fontPtrs[pos++] << ",";
+			uncompressed.push_back(width);
+			for (auto j = 0; j < 3 * width; ++j)
+			{
+				uncompressed.push_back(fontData[ptr + 1 + j]);
+			}
 		}
-		file << std::endl;
 	}
-	file << afterFontPtrs << std::endl;
+
-	for(size_t i = 0; i < fontRanges.size() - 1; i++)
+	std::vector<char> compressed;
-		file << "    { 0x" << std::setw(6) << (unsigned int)fontRanges[i][0] << ", 0x" << std::setw(6) << (unsigned int)fontRanges[i][1] << " }," << std::endl;
+	if (BZ2WCompress(compressed, uncompressed.data(), uncompressed.size()) != BZ2WCompressOk)
-	file << "    { 0, 0 },";
+	{
-	file << afterFontRanges;
+		throw std::runtime_error("Could not compress font data");
+	}
+	file.write(compressed.data(), compressed.size());
+
 	file.close();
 }
 

src/gui/font/FontEditor.h

@@ -5,8 +5,7 @@
 #include <array>
 #include <map>
 
-#include "font.h"
+#include "graphics/FontReader.h"
-
 #include "gui/interface/Window.h"
 
 namespace ui

src/gui/interface/Label.cpp

@@ -8,6 +8,7 @@
 #include "ContextMenu.h"
 
 #include "graphics/Graphics.h"
+#include "graphics/FontReader.h"
 
 using namespace ui;
 

src/gui/interface/TextWrapper.h

@@ -2,7 +2,6 @@
 
 #include "common/String.h"
 #include "Point.h"
-#include "font.h"
 
 #include <vector>