From e55bf48b94365841a010c28a5977c4d9fe65d27f Mon Sep 17 00:00:00 2001
From: apocelipes <seve3r@outlook.com>
Date: Sun, 3 Dec 2023 10:52:39 +0900
Subject: [PATCH] optimize(IO): reduce memory usage during reading data from
files to FFstrbufs
Avoid unnecessary calls of ffStrbufEnsureFree and only allocate
necessary memory.
In most cases, this will save 30% ~ 50% of memory used by ffAppendFDBuffer.
---
src/common/io/io_unix.c | 19 ++++++++++--
src/common/io/io_windows.c | 15 ++++++++--
src/util/FFstrbuf.c | 24 ++++++++++++++++
src/util/FFstrbuf.h | 1 +
tests/strbuf.c | 59 ++++++++++++++++++++++++++++++++++++++
5 files changed, 113 insertions(+), 5 deletions(-)
diff --git a/src/common/io/io_unix.c b/src/common/io/io_unix.c
index 87831e3ed3..c3f592e69e 100644
--- a/src/common/io/io_unix.c
+++ b/src/common/io/io_unix.c
@@ -50,14 +50,29 @@ bool ffAppendFDBuffer(int fd, FFstrbuf* buffer)
if(fstat(fd, &fileInfo) != 0)
return false;
- ffStrbufEnsureFree(buffer, fileInfo.st_size > 0 ? (uint32_t)fileInfo.st_size : 31);
+ if (fileInfo.st_size > 0)
+ {
+ // optimize for files has a fixed length,
+ // file can be very large, only keep necessary memory to save time and resources.
+ ffStrbufEnsureFixedLengthFree(buffer, (uint32_t)fileInfo.st_size);
+ }
+ else
+ ffStrbufEnsureFree(buffer, 31);
uint32_t free = ffStrbufGetFree(buffer);
+ // procfs file's st_size is always zero
+ // choose a signed int type so that can store a native number
+ ssize_t remain = fileInfo.st_size;
while(
(bytesRead = read(fd, buffer->chars + buffer->length, free)) > 0
) {
buffer->length += (uint32_t) bytesRead;
- if((uint32_t) bytesRead == free)
+ // if remain > 0, it means there is some data left in the file.
+ // if remain == 0, it means reading has completed, no need to grow up the buffer.
+ // if remain < 0, we are reading a file from procfs/sysfs and its st_size is zero,
+ // we cannot detect how many data remains in the file, we only can call ffStrbufEnsureFree and read again.
+ remain -= bytesRead;
+ if((uint32_t) bytesRead == free && remain != 0)
ffStrbufEnsureFree(buffer, buffer->allocated - 1); // Doubles capacity every round. -1 for the null byte.
free = ffStrbufGetFree(buffer);
}
diff --git a/src/common/io/io_windows.c b/src/common/io/io_windows.c
index 97f3c80452..f4a3747aa7 100644
--- a/src/common/io/io_windows.c
+++ b/src/common/io/io_windows.c
@@ -37,9 +37,17 @@ bool ffAppendFDBuffer(HANDLE handle, FFstrbuf* buffer)
LARGE_INTEGER fileSize;
if(!GetFileSizeEx(handle, &fileSize))
fileSize.QuadPart = 0;
-
- ffStrbufEnsureFree(buffer, fileSize.QuadPart > 0 ? (uint32_t)fileSize.QuadPart : 31);
+
+ if (fileSize.QuadPart > 0)
+ {
+ // optimize for files has a fixed length,
+ // file can be very large, only keep necessary memory to save time and resources.
+ ffStrbufEnsureFixedLengthFree(buffer, (uint32_t)fileSize.QuadPart);
+ }
+ else
+ ffStrbufEnsureFree(buffer, 31);
uint32_t free = ffStrbufGetFree(buffer);
+ ssize_t remain = fileSize.QuadPart;
bool success;
while(
@@ -47,7 +55,8 @@ bool ffAppendFDBuffer(HANDLE handle, FFstrbuf* buffer)
bytesRead > 0
) {
buffer->length += (uint32_t) bytesRead;
- if((uint32_t) bytesRead == free)
+ remain -= (ssize_t)bytesRead;
+ if((uint32_t) bytesRead == free && remain != 0)
ffStrbufEnsureFree(buffer, buffer->allocated - 1); // Doubles capacity every round. -1 for the null byte.
free = ffStrbufGetFree(buffer);
}
diff --git a/src/util/FFstrbuf.c b/src/util/FFstrbuf.c
index cac5b49a95..574324f7ac 100644
--- a/src/util/FFstrbuf.c
+++ b/src/util/FFstrbuf.c
@@ -54,6 +54,30 @@ void ffStrbufEnsureFree(FFstrbuf* strbuf, uint32_t free)
strbuf->allocated = allocate;
}
+void ffStrbufEnsureFixedLengthFree(FFstrbuf* strbuf, uint32_t free)
+{
+ uint32_t oldFree = ffStrbufGetFree(strbuf);
+ if (oldFree >= free && !(strbuf->allocated == 0 && strbuf->length > 0))
+ return;
+
+ uint32_t newCap = strbuf->allocated + (free - oldFree);
+
+ if(strbuf->allocated == 0)
+ {
+ newCap += strbuf->length + 1; // +1 for the NUL
+ char* newbuf = malloc(sizeof(*strbuf->chars) * newCap);
+ if(strbuf->length == 0)
+ *newbuf = '\0';
+ else
+ memcpy(newbuf, strbuf->chars, strbuf->length + 1);
+ strbuf->chars = newbuf;
+ }
+ else
+ strbuf->chars = realloc(strbuf->chars, sizeof(*strbuf->chars) * newCap);
+
+ strbuf->allocated = newCap;
+}
+
void ffStrbufClear(FFstrbuf* strbuf)
{
assert(strbuf != NULL);
diff --git a/src/util/FFstrbuf.h b/src/util/FFstrbuf.h
index 9a28bafb4e..23859d5f8f 100644
--- a/src/util/FFstrbuf.h
+++ b/src/util/FFstrbuf.h
@@ -33,6 +33,7 @@ void ffStrbufInitA(FFstrbuf* strbuf, uint32_t allocate);
void ffStrbufInitVF(FFstrbuf* strbuf, const char* format, va_list arguments);
void ffStrbufEnsureFree(FFstrbuf* strbuf, uint32_t free);
+void ffStrbufEnsureFixedLengthFree(FFstrbuf* strbuf, uint32_t free);
void ffStrbufClear(FFstrbuf* strbuf);
diff --git a/tests/strbuf.c b/tests/strbuf.c
index 702f405392..e27bc6c2e8 100644
--- a/tests/strbuf.c
+++ b/tests/strbuf.c
@@ -334,6 +334,65 @@ int main(void)
VERIFY(strbuf.allocated > 0);
ffStrbufDestroy(&strbuf);
+ //ffStrbufEnsureFixedLengthFree / empty buffer
+ ffStrbufInit(&strbuf);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+ VERIFY(strbuf.length == 0);
+ VERIFY(strbuf.allocated == 11);
+ ffStrbufDestroy(&strbuf);
+ ffStrbufInitA(&strbuf, 10);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+ VERIFY(strbuf.length == 0);
+ VERIFY(strbuf.allocated == 11);
+ ffStrbufDestroy(&strbuf);
+
+ //ffStrbufEnsureFixedLengthFree / empty buffer with zero free length
+ ffStrbufInit(&strbuf);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 0);
+ VERIFY(strbuf.length == 0);
+ VERIFY(strbuf.allocated == 0);
+ ffStrbufDestroy(&strbuf);
+
+ //ffStrbufEnsureFixedLengthFree / empty buffer but oldFree >= newFree
+ ffStrbufInitA(&strbuf, 11);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+ VERIFY(strbuf.length == 0);
+ VERIFY(strbuf.allocated == 11);
+ ffStrbufDestroy(&strbuf);
+ ffStrbufInitA(&strbuf, 12);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+ VERIFY(strbuf.length == 0);
+ VERIFY(strbuf.allocated == 12);
+ ffStrbufDestroy(&strbuf);
+
+ //ffStrbufEnsureFixedLengthFree / non empty buffer
+ ffStrbufAppendF(&strbuf, "%s", "1234567890");
+ VERIFY(strbuf.length == 10);
+ VERIFY(strbuf.allocated == 32);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 0);
+ VERIFY(strbuf.length == 10);
+ VERIFY(strbuf.allocated == 32);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 20); // less than oldFree (=21)
+ VERIFY(strbuf.length == 10);
+ VERIFY(strbuf.allocated == 32);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 21); // equal to oldFree (=21)
+ VERIFY(strbuf.length == 10);
+ VERIFY(strbuf.allocated == 32);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 22); // greater than oldFree (=21)
+ VERIFY(strbuf.length == 10);
+ VERIFY(strbuf.allocated == 33);
+ ffStrbufDestroy(&strbuf);
+
+ //ffStrbufEnsureFixedLengthFree / static buffer
+ ffStrbufInitStatic(&strbuf, "__TEST__");
+ VERIFY(strbuf.length > 0);
+ VERIFY(strbuf.allocated == 0);
+ ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+ VERIFY(strbuf.length == strlen("__TEST__"));
+ VERIFY(strbuf.allocated == strlen("__TEST__") + 1 + 10);
+ VERIFY(ffStrbufEqualS(&strbuf, "__TEST__"));
+ ffStrbufDestroy(&strbuf);
+
//Success
puts("\033[32mAll tests passed!" FASTFETCH_TEXT_MODIFIER_RESET);
}