[refactoring] FastAPI Set-Up #80

Implemented violence detection

streamlit-fastapi-serving/README.md

@@ -0,0 +1,18 @@
+# FastAPI(BackEnd) <-> Streamlit(FrontEnd) Serving
+ Code composed by transferring most of the operation logic to the backend
+
+
+## How to run
+
+1. install requirements.txt
+
+
+2. FastAPI(BackEnd) run
+```
+uvicorn server:app --host 0.0.0.0 --port 8000
+```
+
+3. Streamlit(FrontEnd) run
+```
+streamlit run ui.py
+```

streamlit-fastapi-serving/fastapi/inference/custom_inference.py

@@ -0,0 +1,69 @@
+from torch.utils.data import DataLoader
+import torch
+import numpy as np
+import time
+import csv
+
+import os, sys
+sys.path.append("..")
+from models.XDVioDet.model import Model
+from models.XDVioDet.dataset import Dataset
+from models.XDVioDet.test import test
+import models.XDVioDet.option as option
+
+
+class ViolenceDetector():
+    def __init__(self, image_list_path, audio_list_path, pretrained_model_path, save_path):
+        self.image_list_path = image_list_path
+        self.audio_list_path = audio_list_path
+        self.pretrained_model_path = pretrained_model_path
+        self.save_path = save_path
+
+    def violence_detection(self):
+        """
+        cwd  /home/user/linux/hipipe/streamlit-fastapi-model-serving-master/fastapi
+        data/list/video.list
+        data/list/audio.list
+        models/XDVioDet/ckpt/wsanodet_mix2.pkl
+        """
+        # args = option.parser.parse_args()
+        args = option.opt
+
+        device = torch.device("cuda")
+
+        test_loader = DataLoader(Dataset(args, test_mode=True),
+                                batch_size=5, shuffle=False,
+                                num_workers=args.workers, pin_memory=True)
+        model = Model(args)
+        model = model.to(device)
+        model_dict = model.load_state_dict(
+            {k.replace('module.', ''): v for k, v in torch.load(self.pretrained_model_path).items()})
+        # gt = np.load(args.gt)
+        st = time.time()
+        pr_auc, pr_auc_online, off, on, index = test(test_loader, model, device)
+        print('Time:{}'.format(time.time()-st))
+        print("[info] Violence Detection Finish")
+        # inference 결과 저장
+        if not os.path.isdir(self.save_path):
+            os.mkdir(self.save_path)
+        np.save(self.save_path+'/off.npy', off)
+        np.save(self.save_path+'/on.npy', on)
+
+        new_index = []
+        for i in index:
+            for j in i:
+                new_index.append(j)
+
+        real_index = []
+        for i in range(0, len(new_index), 5):
+            mp4 = new_index[i][:-7] + '.mp4'
+            mp4 = mp4.split('/')[-1]
+            for j in range(16):
+                real_index.append(mp4)
+        real_index = '\n'.join(real_index)
+
+        file = open('data/list/output_index.list', 'w')
+        file.write(real_index)
+        file.close()
+
+

streamlit-fastapi-serving/fastapi/inference/kinetics_violence_localization.py

@@ -0,0 +1,172 @@
+######## Video Object Detection Using Tensorflow-trained Classifier #########
+#
+# Author: Evan Juras
+# Date: 1/16/18
+# Description:
+# This program uses a TensorFlow-trained classifier to perform object detection.
+# It loads the classifier and uses it to perform object detection on a video.
+# It draws boxes, scores, and labels around the objects of interest in each
+# frame of the video.
+
+## Some of the code is copied from Google's example at
+## https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb
+
+## and some is copied from Dat Tran's example at
+## https://github.com/datitran/object_detector_app/blob/master/object_detection_app.py
+
+## but I changed it to make it more understandable to me.
+
+# Import packages
+import shutil
+import glob
+import os
+import cv2
+from cv2 import threshold
+import numpy as np
+import tensorflow as tf
+import sys
+from collections import deque
+
+# This is needed since the notebook is stored in the object_detection folder.
+# sys.path.append("..")
+tf.compat.v1.disable_v2_behavior()
+# Import utilites
+from ..models.ViolenceDetectionAndLocalization.local_utils import label_map_util
+# from local_utils import visualization_utils as vis_util
+
+# Name of the directory containing the object detection module we're using
+MODEL_NAME = 'inference_graph'
+
+# txtfile = open("hypotheses.txt", "w")
+# Grab path to current working directory
+# CWD_PATH = os.getcwd()
+CWD_PATH = "../models/ViolenceDetectionAndLocalization"
+
+# Path to frozen detection graph .pb file, which contains the model that is used
+# for object detection.
+PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,'frozen_inference_graph.pb')
+
+# Path to label map file
+PATH_TO_LABELS = os.path.join(CWD_PATH,MODEL_NAME,'labelmap.pbtxt')
+
+
+
+# Number of classes the object detector can identify
+NUM_CLASSES = 400
+
+boxfolderspattern='box*'
+videofilespattern='labeledBOX*'
+
+#----------------- Functions
+def cleanfilesandfolders(boxfolderspattern,videofilespattern):
+    # Get a list of all the file paths that ends with .txt from in specified directory
+    fileList = glob.glob(boxfolderspattern)
+    # Iterate over the list of filepaths & remove each file.
+    for filePath in fileList:
+        try:
+            shutil.rmtree(filePath)
+        except:
+            print("Error while deleting folder : ", filePath)
+
+        # Get a list of all the file paths that ends with .txt from in specified directory
+        fileList = glob.glob(videofilespattern)
+        # Iterate over the list of filepaths & remove each file.
+        for filePath in fileList:
+            try:
+                os.remove(filePath)
+            except:
+                print("Error while deleting file : ", filePath)
+#--------------------
+
+
+# Load the label map.
+# Label maps map indices to category names, so that when our convolution
+# network predicts `5`, we know that this corresponds to `king`.
+# Here we use internal utility functions, but anything that returns a
+# dictionary mapping integers to appropriate string labels would be fine
+label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
+categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=False)
+category_index = label_map_util.create_category_index(categories)
+
+# Load the Tensorflow model into memory.
+detection_graph = tf.Graph()
+with detection_graph.as_default():
+    od_graph_def = tf.compat.v1.GraphDef()
+    with tf.io.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
+        serialized_graph = fid.read()
+        od_graph_def.ParseFromString(serialized_graph)
+        tf.import_graph_def(od_graph_def, name='')
+
+    sess = tf.compat.v1.Session(graph=detection_graph)
+
+# Define input and output tensors (i.e. data) for the object detection classifier
+
+# Input tensor is the image
+image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
+
+# Output tensors are the detection boxes, scores, and classes
+# Each box represents a part of the image where a particular object was detected
+detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
+
+# Each score represents level of confidence for each of the objects.
+# The score is shown on the result image, together with the class label.
+detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
+detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
+
+# Number of objects detected
+num_detections = detection_graph.get_tensor_by_name('num_detections:0')
+
+def kinetics_violence_localization(threshold):
+
+    off_path = '../data/npys/off.npy'
+    original_image_path = '../data/images'
+    image_path = '../data/blurred_images'
+    video_name = os.listdir(image_path)[0]
+
+    off_score = np.load(off_path)
+
+    violent_scene = []
+
+    for i in range(len(off_score)):
+        if off_score[i] >= threshold:
+            for j in range(16):
+                violent_scene.append(16*i+j+1)
+
+    if violent_scene:
+        print("Start Localization Blur")
+        for frame_num in violent_scene:
+            # print("Frame %d" % frame_num)
+            img_path = os.path.join(original_image_path, video_name, video_name+'-'+str(frame_num).zfill(6)+".jpg")
+            save_path = os.path.join(image_path, video_name, video_name+'-'+str(frame_num).zfill(6)+".jpg")
+            frame = cv2.imread(img_path)
+            target_img = cv2.imread(save_path)
+            if frame is not None:
+                h,w,c = frame.shape
+                frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                frame_expanded = np.expand_dims(frame_rgb, axis=0)
+
+                # Perform the actual detection by running the model with the image as input
+                (boxes, scores, classes, num) = sess.run(
+                    [detection_boxes, detection_scores, detection_classes, num_detections],
+                    feed_dict={image_tensor: frame_expanded})
+                for box in boxes[0]:
+                    if np.sum(box) == 0:
+                        break
+                    x1,y1,x2,y2 = box
+                    x1 = int(x1*w)
+                    y1 = int(y1*h)
+                    x2 = int(x2*w)
+                    y2 = int(y2*h)
+                    ksize = 25
+                    blur_img = target_img[y1:y2, x1:x2].copy()
+                    target_img[y1:y2, x1:x2] = cv2.blur(blur_img, (ksize,ksize))
+                    cv2.imwrite(save_path, target_img)
+            else:
+                # Clean up
+                print("Stream Ended")
+                # txtfile.close()
+    print("Finish Localization Blur")
+    cleanfilesandfolders(boxfolderspattern, videofilespattern)
+
+# threshold = 0.8
+# kinetics_violence_localization(threshold)

streamlit-fastapi-serving/fastapi/inference/viodet_video.py

@@ -0,0 +1,101 @@
+import os
+import cv2
+import sys
+from PIL import Image
+
+from inference.custom_inference import ViolenceDetector
+sys.path.append("..")
+from utils.file_handler import make_image_from_video, createDirectory, make_feature_list
+from utils.audio2wav import make_wav_from_audio
+from utils.score2figure import make_figure_from_score
+from utils.image_feature_extract import ImageFeatureExtractor
+from utils.audio_feature_extract import AudioFeatureExtractor
+
+
+data_path = "data"
+model_path = "models"
+XDVioDet_path = model_path+"/XDVioDet"
+pytorchi3d_path = model_path+"/pytorchi3d"
+video_root_path = data_path+"/videos"  # feature를 추출할 비디오들이 담긴 폴더 지정
+image_root_path = data_path+"/images"  # jpg가 저장되는 폴더 경로 지정
+createDirectory(image_root_path)
+
+output_directory = data_path+"/audios"  # wav가 저장되는 폴더 경로
+createDirectory(output_directory)
+
+
+def get_violencer():
+    # image feature extract object
+    IFE = ImageFeatureExtractor(
+        root=data_path+"/",
+        mode="rgb",
+        isTrain=False,
+        batch_size=1,
+        save_dir=data_path+"/image_features",
+        load_model=XDVioDet_path+"/ckpt/rgb_imagenet.pt",
+    )
+
+    # audio feature extract object
+    AFE = AudioFeatureExtractor(
+        audio_path=data_path+"/audios", feature_save_path=data_path+"/audio_features"
+    )
+
+    return IFE, AFE
+
+
+def get_violence(model_IFE, model_AFE, threshold):
+    threshold = 0.8 # api 따로 만들어서 처리? or 비디오랑 같이 보내줄 수 있나?
+    # binary_video save 하는 부분 필요 (경로에)
+
+    # mp4 -> jpg
+    make_image_from_video(video_root_path, image_root_path)
+    # mp4 -> wav
+    make_wav_from_audio(video_root_path, output_directory)
+
+    print("[info] image features extracting...")
+    # jpg -> image feature(npy)
+    model_IFE.extract_image_features()
+
+    print("[info] audio features extracting...")
+    # wav -> audio feature(npy)
+    model_AFE.extract_audio_features()
+
+    # npy(image features) -> video.list
+    image_list_path = make_feature_list(
+        target_path=data_path+"/image_features",
+        save_path=data_path+"/list/",
+        file_name="video.list",
+    )
+
+    # npy(audio features) -> audio.list
+    audio_list_path = make_feature_list(
+        target_path=data_path+"/audio_features",
+        save_path=data_path+"/list/",
+        file_name="audio.list",
+    )
+
+    print("[info] Start Violence Detection...")
+
+    # Violence Detection object
+    Detector = ViolenceDetector(
+        image_list_path=image_list_path,
+        audio_list_path=audio_list_path,
+        pretrained_model_path=XDVioDet_path+"/ckpt/wsanodet_mix2.pkl",
+        save_path=data_path+"/npys",
+    )
+    # violence detect
+    Detector.violence_detection()
+
+    # score(npy) -> result.csv
+    make_figure_from_score(
+        threshold,
+        off_path=data_path+"/npys/off.npy",
+        on_path=data_path+"/npys/on.npy",
+        index_path=data_path+"/list/output_index.list",
+        save_path=data_path+"/figures",
+    )
+
+    print(f"[info] Finish Pipeline")
+    violence_score_image = Image.open(data_path+"/figures/score_output_1.png")
+    # return image -> return npy
+    return violence_score_image

streamlit-fastapi-serving/fastapi/models/ViolenceDetectionAndLocalization/README.md

@@ -0,0 +1,17 @@
+# ViolenceDetectionAndLocalization
+ A Violence Detectot with localization , part of A graduation project submited to Birzeit University
+
+ ![](demo.png)
+
+## How to run
+
+1. Download the Pre-trained models Kinetics model and lables and the TensorFlow .pb , .pptxt files from [here](https://u.pcloud.link/publink/show?code=kZlcALXZplDJ4el6eKjNYLPcbHGXsX7qmfV7).
+
+All models are trained on Kinetics-400 and coco dataset. 
+
+2. Extract the Downloaded files to 
+>	| ViolenceDetectionAndLocalization
+> >     resnet-34_kinetics.onnx
+> >		inference_gragh
+> > > 		frozen_inference_graph.pb
+> > > 		labelmap.pbtxt