optimize(IO): reduce memory usage during reading data from files to F…

src/common/io/io_unix.c

@@ -50,14 +50,29 @@ bool ffAppendFDBuffer(int fd, FFstrbuf* buffer)
     if(fstat(fd, &fileInfo) != 0)
         return false;
 
-    ffStrbufEnsureFree(buffer, fileInfo.st_size > 0 ? (uint32_t)fileInfo.st_size : 31);
+    if (fileInfo.st_size > 0)
+    {
+        // optimize for files has a fixed length,
+        // file can be very large, only keep necessary memory to save time and resources.
+        ffStrbufEnsureFixedLengthFree(buffer, (uint32_t)fileInfo.st_size);
+    }
+    else
+        ffStrbufEnsureFree(buffer, 31);
     uint32_t free = ffStrbufGetFree(buffer);
+    // procfs file's st_size is always zero
+    // choose a signed int type so that can store a native number
+    ssize_t remain = fileInfo.st_size;
 
     while(
         (bytesRead = read(fd, buffer->chars + buffer->length, free)) > 0
     ) {
         buffer->length += (uint32_t) bytesRead;
-        if((uint32_t) bytesRead == free)
+        // if remain > 0, it means there is some data left in the file.
+        // if remain == 0, it means reading has completed, no need to grow up the buffer.
+        // if remain < 0, we are reading a file from procfs/sysfs and its st_size is zero,
+        // we cannot detect how many data remains in the file, we only can call ffStrbufEnsureFree and read again.
+        remain -= bytesRead;
+        if((uint32_t) bytesRead == free && remain != 0)
             ffStrbufEnsureFree(buffer, buffer->allocated - 1); // Doubles capacity every round. -1 for the null byte.
         free = ffStrbufGetFree(buffer);
     }

src/common/io/io_windows.c

@@ -37,17 +37,26 @@ bool ffAppendFDBuffer(HANDLE handle, FFstrbuf* buffer)
     LARGE_INTEGER fileSize;
     if(!GetFileSizeEx(handle, &fileSize))
         fileSize.QuadPart = 0;
-
-    ffStrbufEnsureFree(buffer, fileSize.QuadPart > 0 ? (uint32_t)fileSize.QuadPart : 31);
+
+    if (fileSize.QuadPart > 0)
+    {
+        // optimize for files has a fixed length,
+        // file can be very large, only keep necessary memory to save time and resources.
+        ffStrbufEnsureFixedLengthFree(buffer, (uint32_t)fileSize.QuadPart);
+    }
+    else
+        ffStrbufEnsureFree(buffer, 31);
     uint32_t free = ffStrbufGetFree(buffer);
+    ssize_t remain = fileSize.QuadPart;
 
     bool success;
     while(
         (success = !!ReadFile(handle, buffer->chars + buffer->length, free, &bytesRead, NULL)) &&
         bytesRead > 0
     ) {
         buffer->length += (uint32_t) bytesRead;
-        if((uint32_t) bytesRead == free)
+        remain -= (ssize_t)bytesRead;
+        if((uint32_t) bytesRead == free && remain != 0)
             ffStrbufEnsureFree(buffer, buffer->allocated - 1); // Doubles capacity every round. -1 for the null byte.
         free = ffStrbufGetFree(buffer);
     }

src/util/FFstrbuf.c

@@ -54,6 +54,30 @@ void ffStrbufEnsureFree(FFstrbuf* strbuf, uint32_t free)
     strbuf->allocated = allocate;
 }
 
+void ffStrbufEnsureFixedLengthFree(FFstrbuf* strbuf, uint32_t free)
+{
+    uint32_t oldFree = ffStrbufGetFree(strbuf);
+    if (oldFree >= free && !(strbuf->allocated == 0 && strbuf->length > 0))
+        return;
+
+    uint32_t newCap = strbuf->allocated + (free - oldFree);
+
+    if(strbuf->allocated == 0)
+    {
+        newCap += strbuf->length + 1; // +1 for the NUL
+        char* newbuf = malloc(sizeof(*strbuf->chars) * newCap);
+        if(strbuf->length == 0)
+            *newbuf = '\0';
+        else
+            memcpy(newbuf, strbuf->chars, strbuf->length + 1);
+        strbuf->chars = newbuf;
+    }
+    else
+        strbuf->chars = realloc(strbuf->chars, sizeof(*strbuf->chars) * newCap);
+
+    strbuf->allocated = newCap;
+}
+
 void ffStrbufClear(FFstrbuf* strbuf)
 {
     assert(strbuf != NULL);

src/util/FFstrbuf.h

@@ -33,6 +33,7 @@ void ffStrbufInitA(FFstrbuf* strbuf, uint32_t allocate);
 void ffStrbufInitVF(FFstrbuf* strbuf, const char* format, va_list arguments);
 
 void ffStrbufEnsureFree(FFstrbuf* strbuf, uint32_t free);
+void ffStrbufEnsureFixedLengthFree(FFstrbuf* strbuf, uint32_t free);
 
 void ffStrbufClear(FFstrbuf* strbuf);
 

tests/strbuf.c

@@ -334,6 +334,65 @@ int main(void)
     VERIFY(strbuf.allocated > 0);
     ffStrbufDestroy(&strbuf);
 
+    //ffStrbufEnsureFixedLengthFree / empty buffer
+    ffStrbufInit(&strbuf);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+    VERIFY(strbuf.length == 0);
+    VERIFY(strbuf.allocated == 11);
+    ffStrbufDestroy(&strbuf);
+    ffStrbufInitA(&strbuf, 10);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+    VERIFY(strbuf.length == 0);
+    VERIFY(strbuf.allocated == 11);
+    ffStrbufDestroy(&strbuf);
+
+    //ffStrbufEnsureFixedLengthFree / empty buffer with zero free length
+    ffStrbufInit(&strbuf);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 0);
+    VERIFY(strbuf.length == 0);
+    VERIFY(strbuf.allocated == 0);
+    ffStrbufDestroy(&strbuf);
+
+    //ffStrbufEnsureFixedLengthFree / empty buffer but oldFree >= newFree
+    ffStrbufInitA(&strbuf, 11);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+    VERIFY(strbuf.length == 0);
+    VERIFY(strbuf.allocated == 11);
+    ffStrbufDestroy(&strbuf);
+    ffStrbufInitA(&strbuf, 12);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+    VERIFY(strbuf.length == 0);
+    VERIFY(strbuf.allocated == 12);
+    ffStrbufDestroy(&strbuf);
+
+    //ffStrbufEnsureFixedLengthFree / non empty buffer
+    ffStrbufAppendF(&strbuf, "%s", "1234567890");
+    VERIFY(strbuf.length == 10);
+    VERIFY(strbuf.allocated == 32);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 0);
+    VERIFY(strbuf.length == 10);
+    VERIFY(strbuf.allocated == 32);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 20); // less than oldFree (=21)
+    VERIFY(strbuf.length == 10);
+    VERIFY(strbuf.allocated == 32);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 21); // equal to oldFree (=21)
+    VERIFY(strbuf.length == 10);
+    VERIFY(strbuf.allocated == 32);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 22); // greater than oldFree (=21)
+    VERIFY(strbuf.length == 10);
+    VERIFY(strbuf.allocated == 33);
+    ffStrbufDestroy(&strbuf);
+
+    //ffStrbufEnsureFixedLengthFree / static buffer
+    ffStrbufInitStatic(&strbuf, "__TEST__");
+    VERIFY(strbuf.length > 0);
+    VERIFY(strbuf.allocated == 0);
+    ffStrbufEnsureFixedLengthFree(&strbuf, 10);
+    VERIFY(strbuf.length == strlen("__TEST__"));
+    VERIFY(strbuf.allocated == strlen("__TEST__") + 1 + 10);
+    VERIFY(ffStrbufEqualS(&strbuf, "__TEST__"));
+    ffStrbufDestroy(&strbuf);
+
     //Success
     puts("\033[32mAll tests passed!" FASTFETCH_TEXT_MODIFIER_RESET);
 }