Find remaining line from rectangle in OpenCV

Question

I'm detecting a rectangle with known dimensions in OpenCV. I've written code - which works - for detecting all four lines, but the furthest line is difficult to detect and usually causes problems when it comes time for warp the perspective.

My question (which I've looked into, but haven't found an answer for), is whether, theoretically, having 3 lines (angles and intercept) is enough to approximate the final line.

EDIT: I was asked to include my current code. Here it is:

import numpy as np
import cv2
from sklearn.cluster import AgglomerativeClustering

# find intersection of lines
def line_intersect(m1, b1, m2, b2):

    if m1 == m2:
        print ("These lines are parallel!!!")
        return None

    x = int((b2 - b1) / (m1 - m2))

    y = int(m1 * x + b1)

    return [x,y]

# method specs
width_error = 10
height_error = 10
kernel_size = 7
kernel_dilate = np.ones((1, 1), 'uint8')
kernel = np.ones((5, 5), 'uint8')

# read and process image
img = cv2.imread('assets/game-frames/hard-m-2019-124-1200.jpg')
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
blur_gray = cv2.GaussianBlur(gray,(kernel_size, kernel_size),0)
edges = cv2.Canny(blur_gray, 0, 100, apertureSize = 3)
dilate_img = cv2.dilate(edges, kernel_dilate, iterations=3)
closing = cv2.morphologyEx(dilate_img, cv2.MORPH_CLOSE, kernel)

# find hough lines
lines = cv2.HoughLinesP(closing, 2, np.pi/180, 100, minLineLength=20, maxLineGap=10)
lines_np = np.reshape(lines, (np.int32(lines.size/4), 4))
lines_np = lines_np[(lines_np[:, 2] - lines_np[:, 0]) != 0]
lines_np = np.c_[lines_np, (lines_np[:, 3] - lines_np[:, 1])/(lines_np[:, 2] - lines_np[:, 0])]
lines_np = np.c_[lines_np, np.sqrt((lines_np[:, 3] - lines_np[:, 1])**2 + (lines_np[:, 2] - lines_np[:, 0])**2)]

# find lengthwise lines
lengthwise_lines = lines_np[(abs(lines_np[:, 4]) < 3.5) & (abs(lines_np[:, 4]) > 1.20) & (lines_np[:, 5] > 100)]
if lengthwise_lines[:, 4].size < 2: print('No lengthwise lines!')
ward_length = AgglomerativeClustering(n_clusters = None, distance_threshold = 0.50, linkage = "ward").fit(abs(lengthwise_lines[:, 4].reshape(-1,1)))
lengthwise_lines = np.c_[lengthwise_lines, ward_length.labels_]
length_clusts = np.bincount(ward_length.labels_).argsort()[-2:]
lengthwise_lines = lengthwise_lines[np.isin(ward_length.labels_, length_clusts)]

# find extremities
x_max = np.max(lengthwise_lines[:,[0,2]].reshape(-1,1))
x_min = np.amin(lengthwise_lines[:,[0,2]])
y_coords = lengthwise_lines[:, [1, 3]].reshape(-1,1)
ward = AgglomerativeClustering(n_clusters = None, distance_threshold = 10, linkage = "ward").fit(y_coords)
want_clusts = np.bincount(ward.labels_).argsort()[-2:]
y_ext1 = np.median(y_coords[np.isin(ward.labels_, want_clusts[0])])
y_ext2 = np.median(y_coords[np.isin(ward.labels_, want_clusts[1])])
y_max = max(y_ext1, y_ext2)
y_min = min(y_ext1, y_ext2)

# find outer lengthwise lines
length_want = ward_length.labels_[np.isin(ward_length.labels_, length_clusts)][np.argmin(abs(lengthwise_lines[:,4]))]
lengthwise_lines = lengthwise_lines[lengthwise_lines[:, 6] == length_want]
lengthwise_lines = np.c_[lengthwise_lines, (lengthwise_lines[:,1] - (lengthwise_lines[:,4]*lengthwise_lines[:,0]))]
lengthwise_lines = np.c_[lengthwise_lines, ((np.max(lengthwise_lines[:,1]) - lengthwise_lines[:,5])/lengthwise_lines[:,4])]

# subset left and right outer lengthwise lines
l_left = lengthwise_lines[lengthwise_lines[:,4] < 0]
l_right = lengthwise_lines[lengthwise_lines[:,4] > 0]

# find top and bottom lines
widthwise_lines = lines_np[(abs(lines_np[:, 4]) < 0.10)]
widthwise_lines = np.c_[widthwise_lines, (widthwise_lines[:,1] - (widthwise_lines[:,4]*widthwise_lines[:,0]))]
bottom_lines = widthwise_lines[np.where((np.amin(widthwise_lines[:,[1,3]], axis = 1) >= y_min - height_error) & (np.amin(widthwise_lines[:,[1,3]], axis = 1) <= y_min + height_error) & (np.amin(widthwise_lines[:,[0,2]], axis = 1) >= x_min - width_error) & (np.amax(widthwise_lines[:,[0,2]], axis = 1) <= x_max + width_error))[0]]
top_lines = widthwise_lines[np.where((np.amax(widthwise_lines[:,[1,3]], axis = 1) >= y_max - height_error) & (np.amax(widthwise_lines[:,[1,3]], axis = 1) <= y_max + height_error) & (np.amin(widthwise_lines[:,[0,2]], axis = 1) >= x_min - width_error) & (np.amax(widthwise_lines[:,[0,2]], axis = 1) <= x_max + width_error))[0]]

# if no lines found for any border, stop
if top_lines.size == 0 or bottom_lines.size == 0 or l_left.size == 0 or l_right.size == 0: print('No Lengthwise lines!')

# take median of left outer lengthwise lines
l_left_b = np.median(l_left[:,7])
l_left_m = np.median(l_left[:,4])

# take median of right outer lengthwise lines
l_right_b = np.median(l_right[:,7])
l_right_m = np.median(l_right[:,4])

# take median of top widthwise lines
top_lines_b = np.median(top_lines[:,6])
top_lines_m = np.median(top_lines[:,4])

# take median of bottom widthwise lines
bottom_lines_b = np.median(bottom_lines[:,6])
bottom_lines_m = np.median(bottom_lines[:,4])

# find interesction of lines
int_pt1 = line_intersect(l_left_m, l_left_b, bottom_lines_m, bottom_lines_b)
int_pt2 = line_intersect(l_right_m, l_right_b, bottom_lines_m, bottom_lines_b)
int_pt3 = line_intersect(l_left_m, l_left_b, top_lines_m, top_lines_b)
int_pt4 = line_intersect(l_right_m, l_right_b, top_lines_m, top_lines_b)

# draw intersections
cv2.circle(img, int_pt1, 3, (0, 255, 0), -1)
cv2.circle(img, int_pt3, 3, (255, 255, 0), -1)
cv2.circle(img, int_pt2, 3, (0, 255, 0), -1)
cv2.circle(img, int_pt4, 3, (255, 255, 0), -1)

# show image
cv2.imshow('frame diff ', img)
cv2.waitKey(0)

Original image: