diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b2f017c285..9a0e84801f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3562,6 +3562,7 @@ foreach(target ${TARGETS_OWN}) target_compile_definitions(${target} PRIVATE $<$:CONF_DEBUG>) target_include_directories(${target} SYSTEM PRIVATE ${CURL_INCLUDE_DIRS} ${SQLite3_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS}) target_compile_definitions(${target} PRIVATE GLEW_STATIC) + target_compile_definitions(${target} PRIVATE _FILE_OFFSET_BITS=64) # Ensure off_t is 64 bit for ftello and fseeko functions if(CRYPTO_FOUND) target_compile_definitions(${target} PRIVATE CONF_OPENSSL) target_include_directories(${target} SYSTEM PRIVATE ${CRYPTO_INCLUDE_DIRS}) diff --git a/src/base/system.cpp b/src/base/system.cpp index 27afc9cab2e..62d70859aee 100644 --- a/src/base/system.cpp +++ b/src/base/system.cpp @@ -266,12 +266,22 @@ unsigned io_read(IOHANDLE io, void *buffer, unsigned size) return fread(buffer, 1, size, (FILE *)io); } -void io_read_all(IOHANDLE io, void **result, unsigned *result_len) +bool io_read_all(IOHANDLE io, void **result, unsigned *result_len) { - long signed_len = io_length(io); - unsigned len = signed_len < 0 ? 1024 : (unsigned)signed_len; // use default initial size if we couldn't get the length + // Loading files larger than 1 GiB into memory is not supported. + constexpr int64_t MAX_FILE_SIZE = (int64_t)1024 * 1024 * 1024; + + int64_t real_len = io_length(io); + if(real_len > MAX_FILE_SIZE) + { + *result = nullptr; + *result_len = 0; + return false; + } + + int64_t len = real_len < 0 ? 1024 : real_len; // use default initial size if we couldn't get the length char *buffer = (char *)malloc(len + 1); - unsigned read = io_read(io, buffer, len + 1); // +1 to check if the file size is larger than expected + int64_t read = io_read(io, buffer, len + 1); // +1 to check if the file size is larger than expected if(read < len) { buffer = (char *)realloc(buffer, read + 1); @@ -279,7 +289,14 @@ void io_read_all(IOHANDLE io, void **result, unsigned *result_len) } else if(read > len) { - unsigned cap = 2 * read; + int64_t cap = 2 * read; + if(cap > MAX_FILE_SIZE) + { + free(buffer); + *result = nullptr; + *result_len = 0; + return false; + } len = read; buffer = (char *)realloc(buffer, cap); while((read = io_read(io, buffer + len, cap - len)) != 0) @@ -288,6 +305,13 @@ void io_read_all(IOHANDLE io, void **result, unsigned *result_len) if(len == cap) { cap *= 2; + if(cap > MAX_FILE_SIZE) + { + free(buffer); + *result = nullptr; + *result_len = 0; + return false; + } buffer = (char *)realloc(buffer, cap); } } @@ -296,13 +320,17 @@ void io_read_all(IOHANDLE io, void **result, unsigned *result_len) buffer[len] = 0; *result = buffer; *result_len = len; + return true; } char *io_read_all_str(IOHANDLE io) { void *buffer; unsigned len; - io_read_all(io, &buffer, &len); + if(!io_read_all(io, &buffer, &len)) + { + return nullptr; + } if(mem_has_null(buffer, len)) { free(buffer); @@ -311,12 +339,12 @@ char *io_read_all_str(IOHANDLE io) return (char *)buffer; } -int io_skip(IOHANDLE io, int size) +int io_skip(IOHANDLE io, int64_t size) { return io_seek(io, size, IOSEEK_CUR); } -int io_seek(IOHANDLE io, int offset, int origin) +int io_seek(IOHANDLE io, int64_t offset, ESeekOrigin origin) { int real_origin; switch(origin) @@ -334,20 +362,33 @@ int io_seek(IOHANDLE io, int offset, int origin) dbg_assert(false, "origin invalid"); return -1; } - return fseek((FILE *)io, offset, real_origin); +#if defined(CONF_FAMILY_WINDOWS) + return _fseeki64((FILE *)io, offset, real_origin); +#else + return fseeko((FILE *)io, offset, real_origin); +#endif } -long int io_tell(IOHANDLE io) +int64_t io_tell(IOHANDLE io) { - return ftell((FILE *)io); +#if defined(CONF_FAMILY_WINDOWS) + return _ftelli64((FILE *)io); +#else + return ftello((FILE *)io); +#endif } -long int io_length(IOHANDLE io) +int64_t io_length(IOHANDLE io) { - long int length; - io_seek(io, 0, IOSEEK_END); - length = io_tell(io); - io_seek(io, 0, IOSEEK_START); + if(io_seek(io, 0, IOSEEK_END) != 0) + { + return -1; + } + const int64_t length = io_tell(io); + if(io_seek(io, 0, IOSEEK_START) != 0) + { + return -1; + } return length; } diff --git a/src/base/system.h b/src/base/system.h index cf75ce3092a..39fb784ce6e 100644 --- a/src/base/system.h +++ b/src/base/system.h @@ -237,7 +237,13 @@ enum * @see io_open */ IOFLAG_APPEND = 4, +}; +/** + * @ingroup File-IO + */ +enum ESeekOrigin +{ /** * Start seeking from the beginning of the file. * @@ -294,10 +300,14 @@ unsigned io_read(IOHANDLE io, void *buffer, unsigned size); * @param result Receives the file's remaining contents. * @param result_len Receives the file's remaining length. * + * @return `true` on success, `false` on failure. + * * @remark Does NOT guarantee that there are no internal null bytes. * @remark The result must be freed after it has been used. + * @remark The function will fail if more than 1 GiB of memory would + * have to be allocated. Large files should not be loaded into memory. */ -void io_read_all(IOHANDLE io, void **result, unsigned *result_len); +bool io_read_all(IOHANDLE io, void **result, unsigned *result_len); /** * Reads the rest of the file into a zero-terminated buffer with @@ -312,6 +322,8 @@ void io_read_all(IOHANDLE io, void **result, unsigned *result_len); * @remark Guarantees that there are no internal null bytes. * @remark Guarantees that result will contain zero-termination. * @remark The result must be freed after it has been used. + * @remark The function will fail if more than 1 GiB of memory would + * have to be allocated. Large files should not be loaded into memory. */ char *io_read_all_str(IOHANDLE io); @@ -325,7 +337,7 @@ char *io_read_all_str(IOHANDLE io); * * @return 0 on success. */ -int io_skip(IOHANDLE io, int size); +int io_skip(IOHANDLE io, int64_t size); /** * Seeks to a specified offset in the file. @@ -338,7 +350,7 @@ int io_skip(IOHANDLE io, int size); * * @return `0` on success. */ -int io_seek(IOHANDLE io, int offset, int origin); +int io_seek(IOHANDLE io, int64_t offset, ESeekOrigin origin); /** * Gets the current position in the file. @@ -349,7 +361,7 @@ int io_seek(IOHANDLE io, int offset, int origin); * * @return The current position, or `-1` on failure. */ -long int io_tell(IOHANDLE io); +int64_t io_tell(IOHANDLE io); /** * Gets the total length of the file. Resets cursor to the beginning. @@ -360,7 +372,7 @@ long int io_tell(IOHANDLE io); * * @return The total size, or `-1` on failure. */ -long int io_length(IOHANDLE io); +int64_t io_length(IOHANDLE io); /** * Writes data from a buffer to a file. diff --git a/src/engine/client/client.h b/src/engine/client/client.h index 2e9fab96719..f05d7848613 100644 --- a/src/engine/client/client.h +++ b/src/engine/client/client.h @@ -246,7 +246,7 @@ class CClient : public IClient, public CDemoPlayer::IListener int64_t m_BenchmarkStopTime = 0; CChecksum m_Checksum; - int m_OwnExecutableSize = 0; + int64_t m_OwnExecutableSize = 0; IOHANDLE m_OwnExecutable = 0; // favorite command handling diff --git a/src/engine/gfx/image_loader.cpp b/src/engine/gfx/image_loader.cpp index 79d95956d8d..a60d06f3b9b 100644 --- a/src/engine/gfx/image_loader.cpp +++ b/src/engine/gfx/image_loader.cpp @@ -278,8 +278,13 @@ bool CImageLoader::LoadPng(IOHANDLE File, const char *pFilename, CImageInfo &Ima void *pFileData; unsigned FileDataSize; - io_read_all(File, &pFileData, &FileDataSize); + const bool ReadSuccess = io_read_all(File, &pFileData, &FileDataSize); io_close(File); + if(!ReadSuccess) + { + log_error("png", "failed to read file. filename='%s'", pFilename); + return false; + } CByteBufferReader ImageReader(static_cast(pFileData), FileDataSize); diff --git a/src/engine/shared/demo.cpp b/src/engine/shared/demo.cpp index 161e0d6cc16..23c3a124944 100644 --- a/src/engine/shared/demo.cpp +++ b/src/engine/shared/demo.cpp @@ -26,8 +26,6 @@ static const unsigned char gs_CurVersion = 6; static const unsigned char gs_OldVersion = 3; static const unsigned char gs_Sha256Version = 6; static const unsigned char gs_VersionTickCompression = 5; // demo files with this version or higher will use `CHUNKTICKFLAG_TICK_COMPRESSED` -static const int gs_LengthOffset = 152; -static const int gs_NumMarkersOffset = 176; static const ColorRGBA gs_DemoPrintColor{0.75f, 0.7f, 0.7f, 1.0f}; @@ -121,9 +119,31 @@ int CDemoRecorder::Start(class IStorage *pStorage, class IConsole *pConsole, con } if(m_NoMapData) + { MapSize = 0; + } else if(MapFile) - MapSize = io_length(MapFile); + { + const int64_t MapFileSize = io_length(MapFile); + if(MapFileSize > (int64_t)std::numeric_limits::max()) + { + if(CloseMapFile) + { + io_close(MapFile); + } + MapSize = 0; + if(m_pConsole) + { + char aBuf[32 + IO_MAX_PATH_LENGTH]; + str_format(aBuf, sizeof(aBuf), "Mapfile '%s' too large for demo, recording without it", pMap); + m_pConsole->Print(IConsole::OUTPUT_LEVEL_STANDARD, "demo_recorder", aBuf, gs_DemoPrintColor); + } + } + else + { + MapSize = MapFileSize; + } + } // write header CDemoHeader Header; @@ -147,7 +167,7 @@ int CDemoRecorder::Start(class IStorage *pStorage, class IConsole *pConsole, con io_write(DemoFile, SHA256_EXTENSION.m_aData, sizeof(SHA256_EXTENSION.m_aData)); io_write(DemoFile, &Sha256, sizeof(SHA256_DIGEST)); - if(m_NoMapData) + if(MapSize == 0) { } else if(pMapData) @@ -348,13 +368,13 @@ int CDemoRecorder::Stop(IDemoRecorder::EStopMode Mode, const char *pTargetFilena if(Mode == IDemoRecorder::EStopMode::KEEP_FILE) { // add the demo length to the header - io_seek(m_File, gs_LengthOffset, IOSEEK_START); + io_seek(m_File, offsetof(CDemoHeader, m_aLength), IOSEEK_START); unsigned char aLength[sizeof(int32_t)]; uint_to_bytes_be(aLength, Length()); io_write(m_File, aLength, sizeof(aLength)); // add the timeline markers to the header - io_seek(m_File, gs_NumMarkersOffset, IOSEEK_START); + io_seek(m_File, sizeof(CDemoHeader) + offsetof(CTimelineMarkers, m_aNumTimelineMarkers), IOSEEK_START); unsigned char aNumMarkers[sizeof(int32_t)]; uint_to_bytes_be(aNumMarkers, m_NumTimelineMarkers); io_write(m_File, aNumMarkers, sizeof(aNumMarkers)); @@ -550,7 +570,7 @@ CDemoPlayer::EReadChunkHeaderResult CDemoPlayer::ReadChunkHeader(int *pType, int bool CDemoPlayer::ScanFile() { - const long StartPos = io_tell(m_File); + const int64_t StartPos = io_tell(m_File); m_vKeyFrames.clear(); if(StartPos < 0) return false; @@ -558,7 +578,7 @@ bool CDemoPlayer::ScanFile() int ChunkTick = -1; while(true) { - const long CurrentPos = io_tell(m_File); + const int64_t CurrentPos = io_tell(m_File); if(CurrentPos < 0) { m_vKeyFrames.clear(); @@ -846,7 +866,7 @@ unsigned char *CDemoPlayer::GetMapData(class IStorage *pStorage) if(!m_MapInfo.m_Size) return nullptr; - const long CurSeek = io_tell(m_File); + const int64_t CurSeek = io_tell(m_File); if(CurSeek < 0 || io_seek(m_File, m_MapOffset, IOSEEK_START) != 0) return nullptr; unsigned char *pMapData = (unsigned char *)malloc(m_MapInfo.m_Size); @@ -1165,7 +1185,7 @@ bool CDemoPlayer::GetDemoInfo(IStorage *pStorage, IConsole *pConsole, const char { pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "demo_player", "Demo version incremented, but not by DDNet"); } - if(io_seek(File, -(int)ExtensionUuidSize, IOSEEK_CUR) != 0) + if(io_seek(File, -(int64_t)ExtensionUuidSize, IOSEEK_CUR) != 0) { if(pErrorMessage != nullptr) str_copy(pErrorMessage, "Error rewinding SHA256 extension UUID", ErrorMessageSize); diff --git a/src/engine/shared/demo.h b/src/engine/shared/demo.h index 0230e60239a..cede1d85253 100644 --- a/src/engine/shared/demo.h +++ b/src/engine/shared/demo.h @@ -97,10 +97,10 @@ class CDemoPlayer : public IDemoPlayer // Playback struct SKeyFrame { - long m_Filepos; + int64_t m_Filepos; int m_Tick; - SKeyFrame(long Filepos, int Tick) : + SKeyFrame(int64_t Filepos, int Tick) : m_Filepos(Filepos), m_Tick(Tick) { } @@ -108,7 +108,7 @@ class CDemoPlayer : public IDemoPlayer class IConsole *m_pConsole; IOHANDLE m_File; - long m_MapOffset; + int64_t m_MapOffset; char m_aFilename[IO_MAX_PATH_LENGTH]; char m_aErrorMessage[256]; std::vector m_vKeyFrames; diff --git a/src/engine/shared/storage.cpp b/src/engine/shared/storage.cpp index 8f9b69c9eac..4425b0c0af5 100644 --- a/src/engine/shared/storage.cpp +++ b/src/engine/shared/storage.cpp @@ -559,8 +559,14 @@ class CStorage : public IStorage *pResultLen = 0; return false; } - io_read_all(File, ppResult, pResultLen); + const bool ReadSuccess = io_read_all(File, ppResult, pResultLen); io_close(File); + if(!ReadSuccess) + { + *ppResult = nullptr; + *pResultLen = 0; + return false; + } return true; } diff --git a/src/test/io.cpp b/src/test/io.cpp index dde7c22ec66..e08bd063395 100644 --- a/src/test/io.cpp +++ b/src/test/io.cpp @@ -3,22 +3,24 @@ #include -void TestFileRead(const char *pWritten) +static void TestFileRead(const char *pWritten) { + const int WrittenLength = str_length(pWritten); + + char aBuf[64] = {0}; CTestInfo Info; - char aBuf[512] = {0}; + IOHANDLE File = io_open(Info.m_aFilename, IOFLAG_WRITE); - const int WrittenLength = str_length(pWritten); ASSERT_TRUE(File); EXPECT_EQ(io_write(File, pWritten, WrittenLength), WrittenLength); EXPECT_FALSE(io_close(File)); + File = io_open(Info.m_aFilename, IOFLAG_READ); ASSERT_TRUE(File); EXPECT_EQ(io_read(File, aBuf, sizeof(aBuf)), WrittenLength); - EXPECT_TRUE(mem_comp(aBuf, pWritten, WrittenLength) == 0); + EXPECT_EQ(mem_comp(aBuf, pWritten, WrittenLength), 0); EXPECT_FALSE(io_close(File)); - - fs_remove(Info.m_aFilename); + EXPECT_FALSE(fs_remove(Info.m_aFilename)); } TEST(Io, Read1) @@ -38,6 +40,92 @@ TEST(Io, Read4) TestFileRead("\xef\xbb\xbfxyz"); } +static void TestFileLength(const char *pWritten) +{ + const int WrittenLength = str_length(pWritten); + + CTestInfo Info; + + IOHANDLE File = io_open(Info.m_aFilename, IOFLAG_WRITE); + ASSERT_TRUE(File); + EXPECT_EQ(io_write(File, pWritten, WrittenLength), WrittenLength); + EXPECT_FALSE(io_close(File)); + + File = io_open(Info.m_aFilename, IOFLAG_READ); + ASSERT_TRUE(File); + EXPECT_EQ(io_length(File), WrittenLength); + EXPECT_FALSE(io_close(File)); + EXPECT_FALSE(fs_remove(Info.m_aFilename)); +} + +TEST(Io, Length1) +{ + TestFileLength(""); +} +TEST(Io, Length2) +{ + TestFileLength("abc"); +} +TEST(Io, Length3) +{ + TestFileLength("\xef\xbb\xbf"); +} +TEST(Io, Length4) +{ + TestFileLength("\xef\xbb\xbfxyz"); +} + +TEST(Io, SeekTellSkip) +{ + const char *pWritten1 = "01234567890123456789"; + const int WrittenLength1 = str_length(pWritten1); + const char *pWritten2 = "abc"; + const int WrittenLength2 = str_length(pWritten2); + const char *pWritten3 = "def"; + const int WrittenLength3 = str_length(pWritten3); + const char *pWritten4 = "ghi"; + const int WrittenLength4 = str_length(pWritten4); + const char *pWritten5 = "jkl"; + const int WrittenLength5 = str_length(pWritten5); + const char *pExpectedResult = "01def5ghi9abc3456789jkl"; + const int ExpectedLength = str_length(pExpectedResult); + + char aBuf[64] = {0}; + CTestInfo Info; + + IOHANDLE File = io_open(Info.m_aFilename, IOFLAG_WRITE); + ASSERT_TRUE(File); + EXPECT_EQ(io_write(File, pWritten1, WrittenLength1), WrittenLength1); + EXPECT_FALSE(io_seek(File, -10, IOSEEK_CUR)); + EXPECT_EQ(io_write(File, pWritten2, WrittenLength2), WrittenLength2); + EXPECT_FALSE(io_seek(File, 2, IOSEEK_START)); + EXPECT_EQ(io_write(File, pWritten3, WrittenLength3), WrittenLength3); + EXPECT_FALSE(io_skip(File, 1)); + EXPECT_EQ(io_write(File, pWritten4, WrittenLength4), WrittenLength4); + EXPECT_FALSE(io_seek(File, 0, IOSEEK_END)); + EXPECT_EQ(io_write(File, pWritten5, WrittenLength5), WrittenLength5); + EXPECT_FALSE(io_close(File)); + + File = io_open(Info.m_aFilename, IOFLAG_READ); + ASSERT_TRUE(File); + EXPECT_EQ(io_read(File, aBuf, sizeof(aBuf)), ExpectedLength); + EXPECT_EQ(mem_comp(aBuf, pExpectedResult, ExpectedLength), 0); + EXPECT_FALSE(io_seek(File, -13, IOSEEK_CUR)); + EXPECT_EQ(io_read(File, aBuf, WrittenLength2), WrittenLength2); + EXPECT_EQ(mem_comp(aBuf, pWritten2, WrittenLength2), 0); + EXPECT_FALSE(io_seek(File, 2, IOSEEK_START)); + EXPECT_EQ(io_read(File, aBuf, WrittenLength3), WrittenLength3); + EXPECT_EQ(mem_comp(aBuf, pWritten3, WrittenLength3), 0); + EXPECT_FALSE(io_skip(File, 1)); + EXPECT_EQ(io_read(File, aBuf, WrittenLength4), WrittenLength4); + EXPECT_EQ(mem_comp(aBuf, pWritten4, WrittenLength4), 0); + EXPECT_FALSE(io_seek(File, -3, IOSEEK_END)); + EXPECT_EQ(io_read(File, aBuf, WrittenLength5), WrittenLength5); + EXPECT_EQ(mem_comp(aBuf, pWritten5, WrittenLength5), 0); + EXPECT_FALSE(io_close(File)); + EXPECT_FALSE(fs_remove(Info.m_aFilename)); +} + TEST(Io, CurrentExe) { IOHANDLE CurrentExe = io_current_exe(); @@ -45,6 +133,7 @@ TEST(Io, CurrentExe) EXPECT_GE(io_length(CurrentExe), 1024); io_close(CurrentExe); } + TEST(Io, SyncWorks) { CTestInfo Info;