Using opencv python on a jetson Nano, how to reduce capture latency with high resolution camera?

Question

I am using a Jetson Nano to capture high resolution image (21MP, 5344x4012@9FPS, Grayscale, IMX230) and finding aruco markers in them. Running the cv2.aruco.detectMarkers function takes about 2 seconds with current aruco parameters. When testing my system, I was hoping to get a maximum latency of about 2 seconds. However, every frames that are executed by the program are always 15 seconds late. The latency is constant and won't change during program execution.

My first instinct was that I was using the oldest image from a buffer and I tried remedy this problem with this solution. It's basically making a new thread that empties the camera buffer. I modified it a bit so the thread is running indefinitely and the self.cap.read() function is prioritized with runFlag :

# This interface code is based on this arducam repository:
# https://github.com/ArduCAM/MIPI_Camera/tree/master/Jetson/Jetvariety/example

from utils import ArducamUtils
import time
import cv2
import threading

# bufferless VideoCapture from:
# https://stackoverflow.com/a/69141497
class VideoCapture:
  def __init__(self):
    self.cap = cv2.VideoCapture(0, cv2.CAP_V4L2)
    
    # create arducam utilities
    self.arducam_utils = ArducamUtils(0)
  
    # turn off RGB conversion
    self.cap.set(cv2.CAP_PROP_CONVERT_RGB, 0)

    # set pixel format
    fourcc_cap = cv2.VideoWriter_fourcc(*'Y16 ') # Grayscale
    self.cap.set(cv2.CAP_PROP_FOURCC,fourcc_cap) 
    
    # set height and width for max resolution
    self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 5344)
    self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 4012)

    # set max buffer size to 1 frame to limit latency
    print(self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1))
    
    print("[INFO] " + str(self.arducam_utils.get_pixfmt_cfg()))

    # start thread to empty the camera buffer
    self.runFlag = True
    self.t = threading.Thread(target=self._reader)
    self.t.daemon = True
    self.t.start()

  # grab frames as soon as they are available
  def _reader(self):
    while True:
      time.sleep(1.0/9.0)
      if self.runFlag:
        ret = self.cap.grab()
        print("[INFO] Grabbing...")

  # retrieve latest frame
  def read(self):
    self.runFlag = False
    ret, frame = self.cap.read()
    self.runFlag = True
    return frame

  def display(self):
    while True:
      frame = self.read()
      
      w = self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)
      h = self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
      frame = frame.reshape(int(h), int(w))

      frame = self.arducam_utils.convertGrayScale(frame)
      
      frame = resize(frame, 1280.0)
      # display
      cv2.imshow("Arducam", frame)
      ret = cv2.waitKey(1)
      # press 'q' to exit.
      if ret == ord('q'):
        break

  def getFrame(self):
    frame = self.read()
    
    w = self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)
    h = self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
    frame = frame.reshape(int(h), int(w))
    
    frame = self.arducam_utils.convertGrayScale(frame)

    print('[INFO] New Frame.')
    return frame

def resize(frame, dst_width):
  width = frame.shape[1]
  height = frame.shape[0]
  scale = dst_width * 1.0 / width
  return cv2.resize(frame, (int(scale * width), int(scale * height)))

With this hack/fix, I reduced the latency to about 8 seconds. How can I get 2 seconds latency?