OpenCL refactor and wider platform support (#148)

* Remove reference pass from stack

* For testing on all platforms

* Revert

* shared_ptr to std::vector<cl::Event>>

* Refactor share_ptr

* Always save shared reference on ResetWaitEvents()

* WAIT_COPY should not clear saved share_ptrs

* Debugging shared_ptr to vector<cl::Event>

* std::move() event vector

* Debugging (w/deleter)

* Testing custom deleter

* Testing constructor

* Testing emplace shared_ptr

* Checking wait_refs.clear() calls

* QueueCall refactor; std::move test

include/common/oclengine.hpp

@@ -36,6 +36,7 @@ class OCLDeviceCall;
 class OCLDeviceContext;
 
 typedef std::shared_ptr<OCLDeviceContext> DeviceContextPtr;
+typedef std::shared_ptr<std::vector<cl::Event>> EventVecPtr;
 
 enum OCLAPI {
     OCL_API_UNKNOWN = 0,
@@ -121,7 +122,7 @@ class OCLDeviceContext {
     cl::Context context;
     int context_id;
     cl::CommandQueue queue;
-    std::vector<cl::Event> wait_events;
+    EventVecPtr wait_events;
 
 protected:
     std::recursive_mutex mutex;
@@ -140,14 +141,20 @@ class OCLDeviceContext {
         if (error != CL_SUCCESS) {
             queue = cl::CommandQueue(context, d);
         }
+
+        wait_events =
+            std::shared_ptr<std::vector<cl::Event>>(new std::vector<cl::Event>(), [](std::vector<cl::Event>* vec) {
+                vec->clear();
+                delete vec;
+            });
     }
 
     OCLDeviceCall Reserve(OCLAPI call) { return OCLDeviceCall(mutex, calls[call]); }
 
-    std::vector<cl::Event>& ResetWaitEvents()
+    EventVecPtr ResetWaitEvents()
     {
-        std::vector<cl::Event>& waitVec = wait_events;
-        wait_events = std::vector<cl::Event>();
+        EventVecPtr waitVec = std::move(wait_events);
+        wait_events = std::make_shared<std::vector<cl::Event>>();
         return waitVec;
     }
 

include/qengine_cpu.hpp

@@ -168,7 +168,8 @@ class QEngineCPU : public QEngine, public ParallelFor {
 protected:
     virtual void ResetStateVec(complex* nStateVec);
     virtual complex* AllocStateVec(bitCapInt elemCount, bool doForceAlloc = false);
-    virtual void FreeStateVec() {
+    virtual void FreeStateVec()
+    {
         if (stateVec) {
             free(stateVec);
         }

include/qengine_opencl.hpp

@@ -52,6 +52,7 @@ class QEngineOCL : public QEngine {
     complex* stateVec;
     int deviceID;
     DeviceContextPtr device_context;
+    std::vector<EventVecPtr> wait_refs;
     cl::CommandQueue queue;
     cl::Context context;
     // stateBuffer is allocated as a shared_ptr, because it's the only buffer that will be acted on outside of
@@ -205,7 +206,8 @@ class QEngineOCL : public QEngine {
     void InitOCL(int devID);
     void ResetStateVec(complex* nStateVec, BufferPtr nStateBuffer);
     virtual complex* AllocStateVec(bitCapInt elemCount, bool doForceAlloc = false);
-    virtual void FreeStateVec() {
+    virtual void FreeStateVec()
+    {
         if (stateVec) {
             free(stateVec);
         }
@@ -263,8 +265,11 @@ class QEngineOCL : public QEngine {
     void ApplyM(bitCapInt mask, bitCapInt result, complex nrm);
 
     /* Utility functions used by the operations above. */
-    cl::Event QueueCall(OCLAPI api_call, size_t workItemCount, size_t localGroupSize, std::vector<BufferPtr> args,
+    void QueueCall(OCLAPI api_call, size_t workItemCount, size_t localGroupSize, std::vector<BufferPtr> args,
+        size_t localBuffSize = 0);
+    void WaitCall(OCLAPI api_call, size_t workItemCount, size_t localGroupSize, std::vector<BufferPtr> args,
         size_t localBuffSize = 0);
+    EventVecPtr ResetWaitEvents();
     void ApplyMx(OCLAPI api_call, bitCapInt* bciArgs, complex nrm);
     real1 Probx(OCLAPI api_call, bitCapInt* bciArgs);
     void ROx(OCLAPI api_call, bitLenInt shift, bitLenInt start, bitLenInt length);