Support reading demo files larger than 2 GiB

Use `int64_t` for file sizes and offsets of `io_seek`, `io_tell`, `io_skip` and `io_length` functions to support reading larger files, in particular Teehistorian demos larger than 2 GiB. Implement the `io_seek` function using the `_fseeki64` function on Windows and the `fseeko` function on non-Windows. Implement the `io_tell` function using the `_ftelli64` function on Windows and the `ftello` function on non-Windows. Define `_FILE_OFFSET_BITS=64` to ensure that `off_t` is a 64 bit type for the `fseeko` and `ftello` functions.

Change `io_read_all`, `io_read_all_str`, `IStorage::ReadFile` and `IStorage::ReadFileStr` functions to fail when loading files larger than 1 GiB. We should never load files that large into memory and this avoids issues with the file size exceeding `unsigned` limits.

When writing map file data to demos, check if the map file is larger than `std::numeric_limits<unsigned>::max()` which would break the demo format. In that case, record the demo without embedding the map data.

Add tests for the `io_length`, `io_seek`, `io_tell`, `io_skip` functions.

CMakeLists.txt

@@ -3562,6 +3562,7 @@ foreach(target ${TARGETS_OWN})
   target_compile_definitions(${target} PRIVATE $<$<CONFIG:Debug>:CONF_DEBUG>)
   target_include_directories(${target} SYSTEM PRIVATE ${CURL_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
   target_compile_definitions(${target} PRIVATE GLEW_STATIC)
+  target_compile_definitions(${target} PRIVATE _FILE_OFFSET_BITS=64) # Ensure off_t is 64 bit for ftello and fseeko functions
   if(CRYPTO_FOUND)
     target_compile_definitions(${target} PRIVATE CONF_OPENSSL)
     target_include_directories(${target} SYSTEM PRIVATE ${CRYPTO_INCLUDE_DIRS})

src/base/system.cpp

@@ -266,20 +266,37 @@ unsigned io_read(IOHANDLE io, void *buffer, unsigned size)
 	return fread(buffer, 1, size, (FILE *)io);
 }
 
-void io_read_all(IOHANDLE io, void **result, unsigned *result_len)
+bool io_read_all(IOHANDLE io, void **result, unsigned *result_len)
 {
-	long signed_len = io_length(io);
-	unsigned len = signed_len < 0 ? 1024 : (unsigned)signed_len; // use default initial size if we couldn't get the length
+	// Loading files larger than 1 GiB into memory is not supported.
+	constexpr int64_t MAX_FILE_SIZE = (int64_t)1024 * 1024 * 1024;
+
+	int64_t real_len = io_length(io);
+	if(real_len > MAX_FILE_SIZE)
+	{
+		*result = nullptr;
+		*result_len = 0;
+		return false;
+	}
+
+	int64_t len = real_len < 0 ? 1024 : real_len; // use default initial size if we couldn't get the length
 	char *buffer = (char *)malloc(len + 1);
-	unsigned read = io_read(io, buffer, len + 1); // +1 to check if the file size is larger than expected
+	int64_t read = io_read(io, buffer, len + 1); // +1 to check if the file size is larger than expected
 	if(read < len)
 	{
 		buffer = (char *)realloc(buffer, read + 1);
 		len = read;
 	}
 	else if(read > len)
 	{
-		unsigned cap = 2 * read;
+		int64_t cap = 2 * read;
+		if(cap > MAX_FILE_SIZE)
+		{
+			free(buffer);
+			*result = nullptr;
+			*result_len = 0;
+			return false;
+		}
 		len = read;
 		buffer = (char *)realloc(buffer, cap);
 		while((read = io_read(io, buffer + len, cap - len)) != 0)
@@ -288,6 +305,13 @@ void io_read_all(IOHANDLE io, void **result, unsigned *result_len)
 			if(len == cap)
 			{
 				cap *= 2;
+				if(cap > MAX_FILE_SIZE)
+				{
+					free(buffer);
+					*result = nullptr;
+					*result_len = 0;
+					return false;
+				}
 				buffer = (char *)realloc(buffer, cap);
 			}
 		}
@@ -296,13 +320,17 @@ void io_read_all(IOHANDLE io, void **result, unsigned *result_len)
 	buffer[len] = 0;
 	*result = buffer;
 	*result_len = len;
+	return true;
 }
 
 char *io_read_all_str(IOHANDLE io)
 {
 	void *buffer;
 	unsigned len;
-	io_read_all(io, &buffer, &len);
+	if(!io_read_all(io, &buffer, &len))
+	{
+		return nullptr;
+	}
 	if(mem_has_null(buffer, len))
 	{
 		free(buffer);
@@ -311,12 +339,12 @@ char *io_read_all_str(IOHANDLE io)
 	return (char *)buffer;
 }
 
-int io_skip(IOHANDLE io, int size)
+int io_skip(IOHANDLE io, int64_t size)
 {
 	return io_seek(io, size, IOSEEK_CUR);
 }
 
-int io_seek(IOHANDLE io, int offset, int origin)
+int io_seek(IOHANDLE io, int64_t offset, ESeekOrigin origin)
 {
 	int real_origin;
 	switch(origin)
@@ -334,20 +362,33 @@ int io_seek(IOHANDLE io, int offset, int origin)
 		dbg_assert(false, "origin invalid");
 		return -1;
 	}
-	return fseek((FILE *)io, offset, real_origin);
+#if defined(CONF_FAMILY_WINDOWS)
+	return _fseeki64((FILE *)io, offset, real_origin);
+#else
+	return fseeko((FILE *)io, offset, real_origin);
+#endif
 }
 
-long int io_tell(IOHANDLE io)
+int64_t io_tell(IOHANDLE io)
 {
-	return ftell((FILE *)io);
+#if defined(CONF_FAMILY_WINDOWS)
+	return _ftelli64((FILE *)io);
+#else
+	return ftello((FILE *)io);
+#endif
 }
 
-long int io_length(IOHANDLE io)
+int64_t io_length(IOHANDLE io)
 {
-	long int length;
-	io_seek(io, 0, IOSEEK_END);
-	length = io_tell(io);
-	io_seek(io, 0, IOSEEK_START);
+	if(io_seek(io, 0, IOSEEK_END) != 0)
+	{
+		return -1;
+	}
+	const int64_t length = io_tell(io);
+	if(io_seek(io, 0, IOSEEK_START) != 0)
+	{
+		return -1;
+	}
 	return length;
 }
 

src/base/system.h

@@ -237,7 +237,13 @@ enum
 	 * @see io_open
 	 */
 	IOFLAG_APPEND = 4,
+};
 
+/**
+ * @ingroup File-IO
+ */
+enum ESeekOrigin
+{
 	/**
 	 * Start seeking from the beginning of the file.
 	 *
@@ -294,10 +300,14 @@ unsigned io_read(IOHANDLE io, void *buffer, unsigned size);
  * @param result Receives the file's remaining contents.
  * @param result_len Receives the file's remaining length.
  *
+ * @return `true` on success, `false` on failure.
+ *
  * @remark Does NOT guarantee that there are no internal null bytes.
  * @remark The result must be freed after it has been used.
+ * @remark The function will fail if more than 1 GiB of memory would
+ * have to be allocated. Large files should not be loaded into memory.
  */
-void io_read_all(IOHANDLE io, void **result, unsigned *result_len);
+bool io_read_all(IOHANDLE io, void **result, unsigned *result_len);
 
 /**
  * Reads the rest of the file into a zero-terminated buffer with
@@ -312,6 +322,8 @@ void io_read_all(IOHANDLE io, void **result, unsigned *result_len);
  * @remark Guarantees that there are no internal null bytes.
  * @remark Guarantees that result will contain zero-termination.
  * @remark The result must be freed after it has been used.
+ * @remark The function will fail if more than 1 GiB of memory would
+ * have to be allocated. Large files should not be loaded into memory.
  */
 char *io_read_all_str(IOHANDLE io);
 
@@ -325,7 +337,7 @@ char *io_read_all_str(IOHANDLE io);
  *
  * @return 0 on success.
  */
-int io_skip(IOHANDLE io, int size);
+int io_skip(IOHANDLE io, int64_t size);
 
 /**
  * Seeks to a specified offset in the file.
@@ -338,7 +350,7 @@ int io_skip(IOHANDLE io, int size);
  *
  * @return `0` on success.
  */
-int io_seek(IOHANDLE io, int offset, int origin);
+int io_seek(IOHANDLE io, int64_t offset, ESeekOrigin origin);
 
 /**
  * Gets the current position in the file.
@@ -349,7 +361,7 @@ int io_seek(IOHANDLE io, int offset, int origin);
  *
  * @return The current position, or `-1` on failure.
  */
-long int io_tell(IOHANDLE io);
+int64_t io_tell(IOHANDLE io);
 
 /**
  * Gets the total length of the file. Resets cursor to the beginning.
@@ -360,7 +372,7 @@ long int io_tell(IOHANDLE io);
  *
  * @return The total size, or `-1` on failure.
  */
-long int io_length(IOHANDLE io);
+int64_t io_length(IOHANDLE io);
 
 /**
  * Writes data from a buffer to a file.

src/engine/client/client.h

@@ -246,7 +246,7 @@ class CClient : public IClient, public CDemoPlayer::IListener
 	int64_t m_BenchmarkStopTime = 0;
 
 	CChecksum m_Checksum;
-	int m_OwnExecutableSize = 0;
+	int64_t m_OwnExecutableSize = 0;
 	IOHANDLE m_OwnExecutable = 0;
 
 	// favorite command handling

src/engine/gfx/image_loader.cpp

@@ -278,8 +278,13 @@ bool CImageLoader::LoadPng(IOHANDLE File, const char *pFilename, CImageInfo &Ima
 
 	void *pFileData;
 	unsigned FileDataSize;
-	io_read_all(File, &pFileData, &FileDataSize);
+	const bool ReadSuccess = io_read_all(File, &pFileData, &FileDataSize);
 	io_close(File);
+	if(!ReadSuccess)
+	{
+		log_error("png", "failed to read file. filename='%s'", pFilename);
+		return false;
+	}
 
 	CByteBufferReader ImageReader(static_cast<const uint8_t *>(pFileData), FileDataSize);
 

src/engine/shared/demo.cpp

@@ -26,8 +26,6 @@ static const unsigned char gs_CurVersion = 6;
 static const unsigned char gs_OldVersion = 3;
 static const unsigned char gs_Sha256Version = 6;
 static const unsigned char gs_VersionTickCompression = 5; // demo files with this version or higher will use `CHUNKTICKFLAG_TICK_COMPRESSED`
-static const int gs_LengthOffset = 152;
-static const int gs_NumMarkersOffset = 176;
 
 static const ColorRGBA gs_DemoPrintColor{0.75f, 0.7f, 0.7f, 1.0f};
 
@@ -121,9 +119,31 @@ int CDemoRecorder::Start(class IStorage *pStorage, class IConsole *pConsole, con
 	}
 
 	if(m_NoMapData)
+	{
 		MapSize = 0;
+	}
 	else if(MapFile)
-		MapSize = io_length(MapFile);
+	{
+		const int64_t MapFileSize = io_length(MapFile);
+		if(MapFileSize > (int64_t)std::numeric_limits<unsigned>::max())
+		{
+			if(CloseMapFile)
+			{
+				io_close(MapFile);
+			}
+			MapSize = 0;
+			if(m_pConsole)
+			{
+				char aBuf[32 + IO_MAX_PATH_LENGTH];
+				str_format(aBuf, sizeof(aBuf), "Mapfile '%s' too large for demo, recording without it", pMap);
+				m_pConsole->Print(IConsole::OUTPUT_LEVEL_STANDARD, "demo_recorder", aBuf, gs_DemoPrintColor);
+			}
+		}
+		else
+		{
+			MapSize = MapFileSize;
+		}
+	}
 
 	// write header
 	CDemoHeader Header;
@@ -147,7 +167,7 @@ int CDemoRecorder::Start(class IStorage *pStorage, class IConsole *pConsole, con
 	io_write(DemoFile, SHA256_EXTENSION.m_aData, sizeof(SHA256_EXTENSION.m_aData));
 	io_write(DemoFile, &Sha256, sizeof(SHA256_DIGEST));
 
-	if(m_NoMapData)
+	if(MapSize == 0)
 	{
 	}
 	else if(pMapData)
@@ -348,13 +368,13 @@ int CDemoRecorder::Stop(IDemoRecorder::EStopMode Mode, const char *pTargetFilena
 	if(Mode == IDemoRecorder::EStopMode::KEEP_FILE)
 	{
 		// add the demo length to the header
-		io_seek(m_File, gs_LengthOffset, IOSEEK_START);
+		io_seek(m_File, offsetof(CDemoHeader, m_aLength), IOSEEK_START);
 		unsigned char aLength[sizeof(int32_t)];
 		uint_to_bytes_be(aLength, Length());
 		io_write(m_File, aLength, sizeof(aLength));
 
 		// add the timeline markers to the header
-		io_seek(m_File, gs_NumMarkersOffset, IOSEEK_START);
+		io_seek(m_File, sizeof(CDemoHeader) + offsetof(CTimelineMarkers, m_aNumTimelineMarkers), IOSEEK_START);
 		unsigned char aNumMarkers[sizeof(int32_t)];
 		uint_to_bytes_be(aNumMarkers, m_NumTimelineMarkers);
 		io_write(m_File, aNumMarkers, sizeof(aNumMarkers));
@@ -550,15 +570,15 @@ CDemoPlayer::EReadChunkHeaderResult CDemoPlayer::ReadChunkHeader(int *pType, int
 
 bool CDemoPlayer::ScanFile()
 {
-	const long StartPos = io_tell(m_File);
+	const int64_t StartPos = io_tell(m_File);
 	m_vKeyFrames.clear();
 	if(StartPos < 0)
 		return false;
 
 	int ChunkTick = -1;
 	while(true)
 	{
-		const long CurrentPos = io_tell(m_File);
+		const int64_t CurrentPos = io_tell(m_File);
 		if(CurrentPos < 0)
 		{
 			m_vKeyFrames.clear();
@@ -846,7 +866,7 @@ unsigned char *CDemoPlayer::GetMapData(class IStorage *pStorage)
 	if(!m_MapInfo.m_Size)
 		return nullptr;
 
-	const long CurSeek = io_tell(m_File);
+	const int64_t CurSeek = io_tell(m_File);
 	if(CurSeek < 0 || io_seek(m_File, m_MapOffset, IOSEEK_START) != 0)
 		return nullptr;
 	unsigned char *pMapData = (unsigned char *)malloc(m_MapInfo.m_Size);
@@ -1165,7 +1185,7 @@ bool CDemoPlayer::GetDemoInfo(IStorage *pStorage, IConsole *pConsole, const char
 			{
 				pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "demo_player", "Demo version incremented, but not by DDNet");
 			}
-			if(io_seek(File, -(int)ExtensionUuidSize, IOSEEK_CUR) != 0)
+			if(io_seek(File, -(int64_t)ExtensionUuidSize, IOSEEK_CUR) != 0)
 			{
 				if(pErrorMessage != nullptr)
 					str_copy(pErrorMessage, "Error rewinding SHA256 extension UUID", ErrorMessageSize);

src/engine/shared/demo.h

@@ -97,18 +97,18 @@ class CDemoPlayer : public IDemoPlayer
 	// Playback
 	struct SKeyFrame
 	{
-		long m_Filepos;
+		int64_t m_Filepos;
 		int m_Tick;
 
-		SKeyFrame(long Filepos, int Tick) :
+		SKeyFrame(int64_t Filepos, int Tick) :
 			m_Filepos(Filepos), m_Tick(Tick)
 		{
 		}
 	};
 
 	class IConsole *m_pConsole;
 	IOHANDLE m_File;
-	long m_MapOffset;
+	int64_t m_MapOffset;
 	char m_aFilename[IO_MAX_PATH_LENGTH];
 	char m_aErrorMessage[256];
 	std::vector<SKeyFrame> m_vKeyFrames;

src/engine/shared/storage.cpp

@@ -559,8 +559,14 @@ class CStorage : public IStorage
 			*pResultLen = 0;
 			return false;
 		}
-		io_read_all(File, ppResult, pResultLen);
+		const bool ReadSuccess = io_read_all(File, ppResult, pResultLen);
 		io_close(File);
+		if(!ReadSuccess)
+		{
+			*ppResult = nullptr;
+			*pResultLen = 0;
+			return false;
+		}
 		return true;
 	}