YOLOV2 slow when Arduino serial comm inserted

Question

I have been working on object detection and tracking system for a while now. I have tried lighting up LEDs when a person is detected determining the coordinate of the bounding box based on the width of resolution range. As of now, the FPS was around 30 when I did not inserted serial comm functions. But when I have inserted the serial comms, then fps goes down way too low around 7-10. What could be causing the problem here?

OS = Windows

GPU = GTX 1070

CPU = i7

Model = Darkflow, yolov2

Code for object detection.

import cv2
from darkflow.net.build import TFNet
import numpy as np
import time
from collections import namedtuple
import luggage_arduino

"""
Main system for running the whole script for object detection and tracking
"""
class NeuralNetwork:
def __init__(self):
    """Define model configuration and weight"""
    options = {
        'model': 'cfg/yolov2.cfg',
        'load': 'bin/yolov2.weights',
        'threshold': 0.8,  # Sets the confidence level of detecting box, range from 0 to 1
        'gpu': 0.8  # If do not want to use gpu, set to 0
    }

    """Define OpenCV configuration"""
    tfnet = TFNet(options)
    colors = [tuple(255 * np.random.rand(3)) for _ in range(10)]  # Set colors for different boxes
    capture = cv2.VideoCapture(0, cv2.CAP_DSHOW)
    capture.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
    capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)

    while True:  # Main loop for object detection and tracking
        stime = time.time()
        ret, frame = capture.read()
        box = cv2.rectangle(frame, (0, 0), (426, 720), (0, 0, 255), 2)  # Parameter of first segment (LEFT)
        box2 = cv2.rectangle(frame, (426, 0), (852, 720), (0, 0, 255), 2)  # Parameter of second segment (CENTER)
        box3 = cv2.rectangle(frame, (852, 0), (1280, 720), (0, 0, 255), 2)  # Parameter of third segment (RIGHT)
        if ret:
            results = tfnet.return_predict(frame)
            for color, result in zip(colors, results):
                tl = (result['topleft']['x'], result['topleft']['y'])
                br = (result['bottomright']['x'], result['bottomright']['y'])
                label = result['label']
                confidence = result['confidence']
                text = '{}: {:.0f}%'.format(label, confidence * 100)
                frame = cv2.rectangle(frame, tl, br, color, 5)
                frame = cv2.putText(frame, text, tl, cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)
                self.center_of_box(tl, br)  # Calls the function for coordinate calculation
            cv2.imshow('frame', frame)
            print('FPS {:.1f}'.format(1 / (time.time() - stime)))
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    capture.release()
    cv2.destroyAllWindows()

def center_of_box(self, tl, br):
    self.tl = tl
    self.br = br
    center_coord = namedtuple("center_coord", ['x', 'y'])  # List of calculated center coord for each FPS
    center_x = ((tl[0] + br[0]) / 2)
    center_y = ((tl[1] + br[1]) / 2)
    center_box = center_coord(center_x, center_y)  # Save center coord of detected box in list
    print(center_box)
    self.box_tracking(center_x)  # Call function for tracking the box coord

def box_tracking(self, center_x):
    self.center_x = center_x
    while True:
        if 0 <= center_x <= 426:
            center = -1
        elif 426 < center_x <= 852:
            center = 0
        elif 852 < center_x <= 1280:
            center = 1
        else:
            center = 2
        break
    luggage_arduino.Arduino(center)  # Calls function for serial comm

Code for pyserial comms:

import serial
import time

arduino = serial.Serial("com3", 9600)


def serial_comm():  # Pass the function
    pass


"""Main class for serial comm"""


class Arduino:
    def __init__(self, center):
        self.serial_comm(center)  # Calls function of serial comm

    def serial_comm(self, center):
        if center == -1:
            time.sleep(1)
            arduino.write(b'L')  # b can be replaced with str.encode("Your string here")
            serial_comm()
        elif center == 0:
            time.sleep(1)
            arduino.write(b'C')
            serial_comm()
        elif center == 1:
            time.sleep(1)
            arduino.write(b'R')
            serial_comm()
        else:
            center = 2
            time.sleep(1)
            arduino.write(b'N')
            serial_comm()
        time.sleep(2)

@hcheung i used pcutils to check for the rate and it shows 12% and task manager around 10% just when it detects the box and lights the led. — , Dec 25 '19 at 03:55
The calls to `time.sleep()`? Is `class Arduino` used in a `Thread`? When you are using `Serial.readline()`, these calls are blocking and waiting for data. See section Speed Improvement in https://stackoverflow.com/a/59472193/7919597 — Joe, Dec 25 '19 at 06:11
@Joe I havent used threading but im going to learn that and run concurrently. Thank you! — , Dec 26 '19 at 01:36
@Joe please state your comment as answer so i can mark it. Thank you. — , Dec 26 '19 at 02:34

score 0 · Answer 1 · answered Dec 26 '19 at 02:44

For those who are interested, i will post the answer as i figured it out. The solution is to use threading and run the classes concurrently. Here's the code!

import cv2
from darkflow.net.build import TFNet
import numpy as np
import time
from collections import namedtuple
import luggage_arduino
import psutil
import threading

"""
Main system for running the whole script for object detection and tracking
"""
center_of_x = []


class NeuralNetwork():
    def __init__(self):
        self.object_detect()

    def object_detect(self):
        options = {
            'model': 'cfg/yolov2.cfg',
            'load': 'bin/yolov2.weights',
            'threshold': 0.8,  # Sets the confidence level of detecting box, range from 0 to 1
            'gpu': 0.8  # If do not want to use gpu, set to 0
        }
        cpu_usage = psutil.cpu_percent(interval=1)
        tfnet = TFNet(options)
        capture = cv2.VideoCapture(0, cv2.CAP_DSHOW)
        capture.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
        capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
        while True:  # Main loop for object detection and tracking
            stime = time.time()
            ret, frame = capture.read()
            box = cv2.rectangle(frame, (0, 0), (426, 720), (0, 0, 255), 2)  # Parameter of first segment (LEFT)
            box2 = cv2.rectangle(frame, (426, 0), (852, 720), (0, 0, 255), 2)  # Parameter of second segment (CENTER)
            box3 = cv2.rectangle(frame, (852, 0), (1280, 720), (0, 0, 255), 2)  # Parameter of third segment (RIGHT)
            if ret:
                results = tfnet.return_predict(frame)
                for result in results:
                    tl = (result['topleft']['x'], result['topleft']['y'])
                    br = (result['bottomright']['x'], result['bottomright']['y'])
                    label = result['label']
                    confidence = result['confidence']
                    text = '{}: {:.0f}%'.format(label, confidence * 100)
                    frame = cv2.rectangle(frame, tl, br, (255, 153, 255), 2)
                    frame = cv2.putText(frame, text, tl, cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)
                    self.center_of_box(tl, br)
                cv2.imshow('frame', frame)
                print('FPS {:.1f}'.format(1 / (time.time() - stime)))
                print(cpu_usage)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break

        capture.release()
        cv2.destroyAllWindows()

    def center_of_box(self, tl, br):
        global center_of_x
        self.tl = tl
        self.br = br
        center_coord = namedtuple("center_coord", ['x', 'y'])  # List of calculated center coord for each FPS
        center_x = ((tl[0] + br[0]) / 2)
        center_y = ((tl[1] + br[1]) / 2)
        center_box = center_coord(center_x, center_y)  # Save center coord of detected box in list
        center_of_x.clear()
        center_of_x.append(center_x)
        print(center_box)
        return center_x


class Ard:
    def __init__(self):
        time.sleep(7)
        while True:
            for i in center_of_x:
                self.box_tracking(i)

    def box_tracking(self, center_x):
        self.center_x = center_x
        while True:
            if 0 <= center_x <= 426:
                center = -1
            elif 426 < center_x <= 852:
                center = 0
            elif 852 < center_x <= 1280:
                center = 1
            else:
                center = 2
            break
        luggage_arduino.Arduino(center)  # Calls function for serial comm


t1 = threading.Thread(target=NeuralNetwork)
t2 = threading.Thread(target=Ard)

t1.start()
t2.start()

and here's the arduino code.

int data;
int LED2=2;
int LED3=3;
int LED4=4;


void setup() {
  // put your setup code here, to run once:
  Serial.begin(9600);
  pinMode(LED2, OUTPUT);
  digitalWrite (LED2, LOW);
  pinMode(LED3, OUTPUT);
  digitalWrite (LED3, LOW);
  pinMode(LED4, OUTPUT);
  digitalWrite (LED4, LOW);
}

void loop() {
  // put your main code here, to run repeatedly:
  while (Serial.available()){
    data = Serial.read();
    if(data == 'L'){
      digitalWrite(LED2, HIGH);
      delay(100);
      digitalWrite(LED2, LOW);
      digitalWrite(LED3, LOW);
      digitalWrite(LED4, LOW);
    }
    else if (data == 'C'){
      digitalWrite(LED3, HIGH);
      delay(100);
      digitalWrite(LED3, LOW);
      digitalWrite(LED2, LOW);
      digitalWrite(LED4, LOW);
    }
    else if (data == 'R'){
      digitalWrite(LED4, HIGH);
      delay(100);
      digitalWrite(LED4, LOW);
      digitalWrite(LED2, LOW);
      digitalWrite(LED3, LOW);
    }
    else if (data == 'N'){
      digitalWrite(LED2, LOW);
      digitalWrite(LED3, LOW);
      digitalWrite(LED4, LOW);
    }
  }
}

and here's the pyserial code.

import serial
import time

arduino = serial.Serial("com3", 9600)


def serial_comm():  # Pass the function
    pass


"""Main class for serial comm"""


class Arduino:
    def __init__(self, center):
        self.serial_comm(center)  # Calls function of serial comm

    def serial_comm(self, center):
        self.center = center
        if center == -1:
            time.sleep(1)
            arduino.write(b'L')  # b can be replaced with str.encode("Your string here")
            serial_comm()
        elif center == 0:
            time.sleep(1)
            arduino.write(b'C')
            serial_comm()
        elif center == 1:
            time.sleep(1)
            arduino.write(b'R')
            serial_comm()
        else:
            center = 2
            time.sleep(1)
            arduino.write(b'N')
            serial_comm()
        time.sleep(2)

score 0 · Accepted Answer · answered Dec 26 '19 at 06:30

The calls to time.sleep() are probably the reason if your function is not running in a thread.

Is class Arduino used in a Thread? If not, it probably should.

When you are using Serial.readline(), these calls are blocking and waiting for data.

See section Speed Improvement in this answer and the code posted here.

YOLOV2 slow when Arduino serial comm inserted

2 Answers2