Fallback from CL to cpu when buffer sizes too large

[dsignarius]

Thank you for developing this, enjoying greatly exploring it.

While playing, have found some instances require a buffer larger than my opencl setup will allow.. mainly due to it only allowing 1/4 size as max buffer, also small vram.

This patch will allow the CL version to fallback to openblas if any of the buffer sizes will be over the size that your machine will allow you to allocate.
This would just fail on the call otherwise, so allows things to carry on. Also allows users with smaller vrams and opencl cards to play around with larger models. It will fit the bits that fit on the card on it, otherwise use the cpu variant.

diff --git a/ggml_blas_adapter.c b/ggml_blas_adapter.c
index 27ddcdb..a73e95f 100644
--- a/ggml_blas_adapter.c
+++ b/ggml_blas_adapter.c
@@ -32,6 +32,13 @@ cl_program program;
cl_kernel kernel_q4_0, kernel_q4_1, kernel_q4_2, kernel_q4_3;
bool cl_initialized = false;

+#define CLFALLBACKTOBLAS
+
+#ifdef CLFALLBACKTOBLAS
+cl_ulong cl_max_mem_size = 0;
+cl_ulong cl_max_buffer_size = 0;
+#endif
+
size_t cl_size_a = 0, cl_size_b = 0, cl_size_qb = 0, cl_size_c = 0;
cl_mem cl_buffer_a, cl_buffer_b, cl_buffer_qb, cl_buffer_c;

@@ -152,6 +159,25 @@ static void ggml_cl_sgemm_wrapper(const enum CBLAS_ORDER order, const enum CBLAS
queue = clCreateCommandQueue(context, device, 0, &err);
CL_CHECK(err, "clCreateCommandQueue");

+#ifdef CLFALLBACKTOBLAS

•    err = clGetDeviceInfo(device, CL_DEVICE_GLOBAL_MEM_SIZE, 
  

•        sizeof(cl_ulong), &cl_max_mem_size, NULL);
  

•    if(err != CL_SUCCESS){
  

•        printf("Error querying OpenCL global memory size : %d\n", err);
  

•        fflush(stdout);
  

•    } 
  

•    err = clGetDeviceInfo(device, CL_DEVICE_MAX_MEM_ALLOC_SIZE, 
  

•        sizeof(cl_ulong), &cl_max_buffer_size, NULL);
  

•    if(err != CL_SUCCESS){
  

•        printf("Error querying OpenCL max alloc size : %d\n", err);
  

•        fflush(stdout);
  

•    } 
  

•    printf("OpenCL maxmem: %ld , maxalloc: %ld\n", cl_max_mem_size,cl_max_buffer_size);
  

•    fflush(stdout);       
  

+#endif
+
+
free(platforms);
free(devices);

@@ -185,6 +211,47 @@ static void ggml_cl_sgemm_wrapper(const enum CBLAS_ORDER order, const enum CBLAS
cl_initialized = true;
}

+#ifdef CLFALLBACKTOBLAS

• cl_ulong buffer_a_size = mksizeof(float);
• cl_ulong buffer_b_size = nksizeof(float);
• cl_ulong buffer_c_size = mnsizeof(float);
• cl_ulong buffer_qb_size = (n*k) * (sizeof(float) * (btype == 2 ? 1 : 2) + 16) / 32;
• cl_ulong buffers_total_size = buffer_a_size + buffer_b_size + buffer_c_size + buffer_qb_size;
• // if any buffer size is more than max CL alloc OR
• // total exceeds CL memory, use non CL version
• if ((buffer_a_size > cl_max_buffer_size) ||

•    (buffer_b_size > cl_max_buffer_size) ||
  

•    (buffer_c_size > cl_max_buffer_size) ||
  

•    (buffer_qb_size > cl_max_buffer_size) ||
  

•    (buffers_total_size > cl_max_mem_size) ) {
  

+#if 1

•    printf("\ncblas Fallback - CL maxalloc: %ld ", cl_max_buffer_size);
  

•    if (buffer_a_size > cl_max_buffer_size)
  

•        printf(" buffer a: %ld ",buffer_a_size);
  

•    if (buffer_b_size > cl_max_buffer_size)
  

•        printf(" buffer b: %ld ",buffer_b_size);
  

•    if (buffer_c_size > cl_max_buffer_size)
  

•        printf(" buffer c: %ld ",buffer_c_size);
  

•    if (buffer_qb_size > cl_max_buffer_size)
  

•        printf(" buffer qb: %ld ",buffer_qb_size);
  

•    printf("\n");
  

•    fflush(stdout);
  

+#endif

•    // Call the SGEMM routine.
  

•    cblas_sgemm(order, 
  

•        trans_a, trans_b, 
  

•        m, n, k, 
  

•        alpha, 
  

•        host_a, lda, host_b, ldb, 
  

•        beta, 
  

•        host_c, ldc);
  

• }
• else
• {
  +#endif
• bool dequant = (btype >= 2 && btype < 6);
  cl_kernel kernel;

@@ -291,7 +358,9 @@ static void ggml_cl_sgemm_wrapper(const enum CBLAS_ORDER order, const enum CBLAS
clReleaseEvent(events[0]);
clReleaseEvent(events[1]);
}

+#ifdef CLFALLBACKTOBLAS

• }
  +#endif
  }
  #endif
  #endif

[LostRuins]

CLblast actually does use very small buffers - should not exceed a couple hundred MB at most. If your GPU cannot handle this , it is probably better to just start with OpenBLAS.

[dsignarius]

i am finding one of 655,380,480 (buffer b) that keeps popping up.. sometimes one per query/reply loop. the others pass and are a lot smaller.
the cl does give a speedup when called, noticeable speedup.. started on openblas, then tried the cl changes... thought a hybrid might help those with laptops, apu's and smaller cards.

and because its per call of this, many times per query/reply loop it does make a difference, imho.