Color integration

Destin/Bindings/Python/letters_experiment.py

@@ -20,7 +20,8 @@
     # CZT
     #
     #ims.addImage("/home/teaera/Downloads/2/%s.png" % l)
-    ims.addImage("/home/teaera/Downloads/destin_toshare/train images/%s.png" % l)
+    #ims.addImage("/home/teaera/Downloads/destin_toshare/train images/%s.png" % l)
+    ims.addImage("letters/%s.png" % l)
 
 #centroids =  [2,2,8,32,40,32,15,len(letters)]
 centroids =  [4,8,16,32,64,32,16,len(letters)]
@@ -38,7 +39,7 @@
 save_root="./saves/"
 
 def train():
-    for i in range(3500):
+    for i in range(3500):#it was 3500
         if i % 10 == 0:
             print "Iteration " + str(i)
 
@@ -60,7 +61,7 @@ def dci(layer, cent, equalize_hist = False, exp_weight = 4):
 
 def dcis(layer):
     dn.displayLayerCentroidImages(layer,1000)
-    cv.WaitKey(100)
+    cv.WaitKey(10000)
 
 
 def mkdir(path):
@@ -139,7 +140,7 @@ def getTreeLabel(layer, cent, child_num):
 
 dn.save("+LO.dst")
 #dci(7,0,False, weight_exponent)
-dcis(7)
+dcis(6)
 
 tm = pd.DestinTreeManager(dn, 0)
 #tm.displayTree()

Destin/Bindings/Python/tree_mining_experiment.py

@@ -121,8 +121,11 @@ def saveResults(run_id):
 
 ims = pd.ImageSouceImpl() 
 
-letters = "+LO"
-img_path = "/home/ted/Pictures/treeminingletters/"
+#letters = "+LO"
+letters = "ABC"
+#img_path = "/home/ted/Pictures/treeminingletters/"
+img_path = "letters/"
+
 for l in letters:    
     ims.addImage("%s%s.png" % (img_path, l))
 
@@ -144,7 +147,7 @@ def saveResults(run_id):
 print "found %i trees" % (found)
 
 matchSubtrees()
-displayCentroidImages(7)    
+displayCentroidImages(7)
 
 saveResults("2")
 print "click on tree images to continue"

Destin/Bindings/Python/tree_mining_runs/2/matches.txt

@@ -0,0 +1,16 @@
+tree #0 matches input image: 0 2 
+tree #1 matches input image: 0 1 2 
+tree #2 matches input image: 1 2 
+tree #3 matches input image: 0 1 
+tree #4 matches input image: 0 2 
+tree #5 matches input image: 1 2 
+tree #6 matches input image: 1 2 
+tree #7 matches input image: 0 1 2 
+tree #8 matches input image: 1 2 
+tree #9 matches input image: 0 1 
+tree #10 matches input image: 1 2 
+tree #11 matches input image: 0 2 
+tree #12 matches input image: 0 1 2 
+tree #13 matches input image: 0 2 
+tree #14 matches input image: 0 1 
+tree #15 matches input image: 0 2 

Destin/Bindings/Python/tree_mining_runs/2/treepath.txt

@@ -0,0 +1,32 @@
+tree #0: size: 1 : (L4,C1,P0)
+
+tree #1: size: 1 : (L4,C9,P0)
+
+tree #2: size: 1 : (L4,C57,P0)
+
+tree #3: size: 1 : (L4,C63,P1)
+
+tree #4: size: 1 : (L4,C2,P2)
+
+tree #5: size: 1 : (L4,C12,P2)
+
+tree #6: size: 1 : (L4,C28,P2)
+
+tree #7: size: 9 : (L5,C0,P0)(L4,C56,P0) (GoUp) (L4,C56,P1) (GoUp) (L4,C56,P2) (GoUp) (L4,C56,P3) (GoUp) 
+
+tree #8: size: 5 : (L5,C24,P0)(L4,C56,P0) (GoUp) (L4,C56,P1) (GoUp) 
+
+tree #9: size: 9 : (L5,C0,P1)(L4,C56,P0) (GoUp) (L4,C56,P1) (GoUp) (L4,C56,P2) (GoUp) (L4,C56,P3) (GoUp) 
+
+tree #10: size: 1 : (L5,C18,P1)
+
+tree #11: size: 5 : (L5,C27,P1)(L4,C56,P0) (GoUp) (L4,C56,P1) (GoUp) 
+
+tree #12: size: 9 : (L5,C0,P2)(L4,C56,P0) (GoUp) (L4,C56,P1) (GoUp) (L4,C56,P2) (GoUp) (L4,C56,P3) (GoUp) 
+
+tree #13: size: 7 : (L5,C5,P2)(L4,C56,P0) (GoUp) (L4,C56,P2) (GoUp) (L4,C56,P3) (GoUp) 
+
+tree #14: size: 9 : (L5,C0,P3)(L4,C56,P0) (GoUp) (L4,C56,P1) (GoUp) (L4,C56,P2) (GoUp) (L4,C56,P3) (GoUp) 
+
+tree #15: size: 7 : (L5,C2,P3)(L4,C56,P1) (GoUp) (L4,C56,P2) (GoUp) (L4,C56,P3) (GoUp) 
+

Destin/Bindings/Python/tree_mining_runs/2/treepath_structure.txt

@@ -0,0 +1,74 @@
+tree #0:
+(L4,C1,P0)
+
+tree #1:
+(L4,C9,P0)
+
+tree #2:
+(L4,C57,P0)
+
+tree #3:
+(L4,C63,P1)
+
+tree #4:
+(L4,C2,P2)
+
+tree #5:
+(L4,C12,P2)
+
+tree #6:
+(L4,C28,P2)
+
+tree #7:
+(L5,C0,P0)
+	(L4,C56,P0)
+	(L4,C56,P1)
+	(L4,C56,P2)
+	(L4,C56,P3)
+
+tree #8:
+(L5,C24,P0)
+	(L4,C56,P0)
+	(L4,C56,P1)
+
+tree #9:
+(L5,C0,P1)
+	(L4,C56,P0)
+	(L4,C56,P1)
+	(L4,C56,P2)
+	(L4,C56,P3)
+
+tree #10:
+(L5,C18,P1)
+
+tree #11:
+(L5,C27,P1)
+	(L4,C56,P0)
+	(L4,C56,P1)
+
+tree #12:
+(L5,C0,P2)
+	(L4,C56,P0)
+	(L4,C56,P1)
+	(L4,C56,P2)
+	(L4,C56,P3)
+
+tree #13:
+(L5,C5,P2)
+	(L4,C56,P0)
+	(L4,C56,P2)
+	(L4,C56,P3)
+
+tree #14:
+(L5,C0,P3)
+	(L4,C56,P0)
+	(L4,C56,P1)
+	(L4,C56,P2)
+	(L4,C56,P3)
+
+tree #15:
+(L5,C2,P3)
+	(L4,C56,P1)
+	(L4,C56,P2)
+	(L4,C56,P3)
+

Destin/CMakeLists.txt

@@ -22,4 +22,4 @@ ADD_SUBDIRECTORY(SOM)
 # 2013.4.11
 # CZT
 #
-ADD_SUBDIRECTORY(czt_test)
+#ADD_SUBDIRECTORY(czt_test)

Destin/Common/DestinNetworkAlt.cpp

@@ -255,6 +255,31 @@ void DestinNetworkAlt::imageWinningCentroidGrid(int layer, int zoom, const strin
     return;
 }
 
+void DestinNetworkAlt::eth_imageWinningCentroidGrid(int layer, int zoom, const string& window_name){
+    uint width = (uint)sqrt(destin->layerSize[layer]);
+
+    //initialize or create new grid if needed
+    if(winningGrid.rows != width || winningGrid.cols != width){
+        //winningGrid = cv::Mat(width, width, CV_32FC1);
+        winningGrid = cv::Mat(width, width, CV_8UC3);
+    }
+
+    int n=0;
+    for(int i=0;i<winningGrid.rows;i++){
+        for(int j=0;j<winningGrid.cols;j++){
+            winningGrid.at<cv::Vec3b>(i,j)[0]= GetNodeFromDestinI(destin, layer, n)->observation[0]*255;
+            winningGrid.at<cv::Vec3b>(i,j)[1]= GetNodeFromDestinI(destin, layer, n)->observation[1]*255;
+            winningGrid.at<cv::Vec3b>(i,j)[2]= GetNodeFromDestinI(destin, layer, n)->observation[2]*255;
+
+            n++;
+        }
+    }
+
+    cv::resize(winningGrid,winningGridLarge, cv::Size(), zoom, zoom, cv::INTER_NEAREST);
+    cv::imshow(window_name, winningGridLarge);
+    return;
+}
+
 void DestinNetworkAlt::printNodeObservation(int layer, int row, int col){
     Node * n = getNode(layer, row, col);
     printf("Node Observation: layer %i, row %i, col: %i\n", layer, row, col);

Destin/Common/DestinNetworkAlt.h

@@ -176,6 +176,8 @@ class DestinNetworkAlt: public INetwork {
       */
     void imageWinningCentroidGrid(int layer, int zoom = 8, const string& window_name="Winning Grid");
 
+    void eth_imageWinningCentroidGrid(int layer, int zoom = 8, const string& window_name="Winning Grid");
+
     void printNodeObservation(int layer, int row, int col);
 
     void printNodeBeliefs(int layer, int row, int col);

Destin/Common/VideoSource.cpp

@@ -75,6 +75,70 @@ bool VideoSource::grab() {
     }
 }
 
+bool VideoSource::eth_grab() {//@eth_function
+    if (cap->grab()) {
+        cap->retrieve(original_frame); //retrieve the captured frame
+
+        original_size = original_frame.size();
+
+        // see http://opencv.willowgarage.com/documentation/cpp/imgproc_geometric_image_transformations.html#resize
+        cv::resize(original_frame, rescaled_frame, target_size ,1.0,1.0); //resize image to target_size
+
+        cv::flip(rescaled_frame, flipped_frame, 1);
+
+        greyscaled_frame=flipped_frame;//@eth : removed the color conversion to grey scale
+        //cvtColor(flipped_frame, greyscaled_frame, CV_BGR2GRAY); //turn the image grey scale @eth
+
+        //convert the greyscaled_frame into a float array for DeSTIN
+        eth_convert(greyscaled_frame, this->float_frame);
+
+        if(edge_detection){
+            cv::GaussianBlur(greyscaled_frame, greyscaled_frame, cv::Size(7,7) , .75, .75); // blur the image
+            cv::Canny(greyscaled_frame, greyscaled_frame, 0, 30, 3); // apply edge detection
+        }
+
+        /*if(showWindow){
+            cv::imshow( DESTIN_VIDEO_WINDOW_TITLE, greyscaled_frame); //show video to output window
+
+        }*/
+
+        // 2013.4.19
+        // CZT
+        //
+        if(showWindow){
+            cv::imshow(this->win_title, greyscaled_frame); //show video to output window
+        }
+
+        // some strange issues with waitkey, see http://opencv.willowgarage.com/wiki/documentation/c/highgui/WaitKey
+        //Needs this so it gives time for the computer to update. If its too small, it wont be drawn at all,
+        //if its too high, then the frames per seconds drops.
+
+        if(showWindow && cv::waitKey(5)>=0){ //stop the video when a key is pressed
+            return false;
+        }
+        return true;
+    } else {
+        //this logic rewinds the video to the beginning if it can.
+        if(cap->get(CV_CAP_PROP_FRAME_COUNT) > 0){
+            if(cap->get(CV_CAP_PROP_POS_AVI_RATIO) > 0.9){   // if reached near end of video
+                cap->set(CV_CAP_PROP_POS_FRAMES,0);         // move to beginning
+                if(cap->grab()){                            // if can get a frame
+                    //cout << "video rewind " << endl;
+                    return this->grab();                    // get the next frame
+                }else{
+                    cout << "couldn't rewind and grab first frame." << endl;
+                }
+            }else{
+                cout << "couldn't get video and not end of video. was: " <<  cap->get(CV_CAP_PROP_POS_AVI_RATIO) << endl;
+            }
+        }else{
+            cout << "frame count not larger than 0\n";
+        }
+
+        cout << "Reached end of video stream." << endl;
+        return false;
+    }
+}
 
 bool VideoSource::rewind(){
      if(cap->get(CV_CAP_PROP_FRAME_COUNT) == 0){

Destin/Common/VideoSource.h

@@ -87,6 +87,35 @@ class VideoSource {
 			}
 		}
 	}
+
+    void eth_convert(cv::Mat & in, float * out) {//@eth_function
+        /*if(in.channels()!=1){
+            throw runtime_error("Excepted a grayscale image with one channel.");
+        }*/
+        if(in.depth()!=CV_8U){
+            throw runtime_error("Expected image to have bit depth of 8bits unsigned integers ( CV_8U )");
+        }
+        cv::Point p(0, 0);
+        int i = 0 ;
+        for (p.y = 0; p.y < in.rows; p.y++) {
+            for (p.x = 0; p.x < in.cols; p.x++) {
+                //i = frame.at<uchar>(p);
+                //use something like frame.at<Vec3b>(p)[channel] in case of trying to support color images.
+                //There would be 3 channels for a color image (one for each of r, g, b)
+
+                //out[i] = (float)in.at<uchar>(p) / 255.0f;
+                //i++;
+
+                //@eth : add two more variables for the RGB color
+                out[i]=(float)in.at<cv::Vec3b>(p)[0] / 255.0f;
+                out[i+(512*512*1)]=(float)in.at<cv::Vec3b>(p)[1] / 255.0f;
+                out[i+(512*512*2)]=(float)in.at<cv::Vec3b>(p)[2] / 255.0f;
+
+                i++;
+
+            }
+        }
+    }
 public:
 	/**
 	 * use_device - if true then will find the default device i.e. webcam as input
@@ -121,6 +150,10 @@ class VideoSource {
         av_log_set_level(AV_LOG_QUIET);//turn off message " No accelerated colorspace conversion found from yuv422p to bgr24"
 	}
 
+    void increaseFrame(int ratio){//@eth_function : increases the size of the frame
+        float_frame = new float[ratio*target_size.area()];
+    }
+
 	bool isOpened() {
 		return cap->isOpened();
 	}
@@ -193,6 +226,8 @@ class VideoSource {
 	 */
 	bool grab();
 
+    bool eth_grab();
+
     /** rewinds the video
      *  @return - true if it did rewind false otherwise
      */

Destin/DavisDestin/src/node.c

@@ -165,7 +165,13 @@ void GetObservation_c1( Node *n, float *framePtr, uint nIdx )
         // If so, use input from the input image
         for( i=0; i < ni; i++ )
         {
-            n->observation[i] = framePtr[n->inputOffsets[i]];
+            //n->observation[i] = framePtr[n->inputOffsets[i]];
+
+            //@eth : add two more inputs for the RGB
+            n->observation[3*i] = framePtr[n->inputOffsets[i]];
+            n->observation[3*i+1] = framePtr[n->inputOffsets[i]+(512*512*1)];
+            n->observation[3*i+2] = framePtr[n->inputOffsets[i]+(512*512*2)];
+
 
             // 2013.4.11
             // CZT

Destin/Misc/CMakeLists.txt

@@ -8,3 +8,6 @@ INCLUDE_DIRECTORIES( ../DavisDestin/include )
 ADD_EXECUTABLE( webcamExp WebcamExperiment ../Common/VideoSource )
 TARGET_LINK_LIBRARiES( webcamExp destinalt ${OpenCV_LIBS} avutil common)
 
+INCLUDE_DIRECTORIES( ../DavisDestin/include )
+ADD_EXECUTABLE( color colorExp ../Common/VideoSource )
+TARGET_LINK_LIBRARiES( color destinalt ${OpenCV_LIBS} avutil common)