I am stumped, maybe I am misunderstanding something so I will post and see what insights come back.
I have a Flask API app (bjoern WSGI) that uses JWT for auth. I send the flask app some credentials and a local URL to hit an API that will provide images. The flask app then runs some ML on the images and replies with the detection data. Once the client receives the response from the flask app, it also requests the matching image from the API (double hit).
What I want to do is pass the matching image from the flask app back to the client along with the detection data in JSON format. I have achieved all of this by using requests-toolbox and a MultiPartEncoder. My problem is when I try and use cv2.imdecode to encode the response.content bytes into a jpeg on the client-side. Here is the pertinent code and I will also post the output and error. I do not want to write the image to disk and read it back into a numpy array, I am trying to do it all in memory.
This is on the client-side, this section is where it sends the request to the flask app.
do_raise = True
try:
from requests_toolbelt.multipart import decoder
r = requests.post(
url=ml_object_url,
headers=auth_header,
params=params,
# json doesnt send when sending files to mlapi, so we send the file and the json together
json=mlapi_json if not files else None,
files=files,
)
r.raise_for_status()
except ValueError as v_ex:
# pass as we will do a retry loop? -> urllib3 has a retry loop built in but idk if that works here
if v_ex == "BAD_IMAGE":
pass
except requests.exceptions.HTTPError as http_ex:
if http_ex.response.status_code == 400:
if args.get('file'):
g.logger.error(
f"There seems to be an error trying to send an image from zm_detect to mlapi, looking into it"
)
else:
g.logger.error(f"{http_ex.response.json()}")
elif http_ex.response.status_code == 500:
g.logger.error(f"There seems to be an Internal Error with the mlapi host, check mlapi logs!")
else:
g.logger.error(f"ERR CODE={http_ex.response.status_code} {http_ex.response.content=}")
except urllib3.exceptions.NewConnectionError as urllib3_ex:
g.logger.debug(f"{lp} {urllib3_ex.args=} {urllib3_ex.pool=}")
g.logger.error(
f"There seems to be an error while trying to start a new connection to the mlapi host -> {urllib3_ex}")
except requests.exceptions.ConnectionError as req_conn_ex:
g.logger.error(
f"There seems to be an error while trying to start a new connection to the mlapi host -> "
f"{req_conn_ex.response}"
)
except Exception as all_ex:
g.logger.error(
f"{lp} error during post to mlapi host-> {all_ex}"
)
# g.logger.debug(f"traceback-> {format_exc()}")
else:
do_raise = False
data: Optional[dict] = None
multipart_data: decoder.MultipartDecoder = decoder.MultipartDecoder.from_response(r)
part: decoder.MultipartDecoder.from_response
img: Optional[bytes] = None
for part in multipart_data.parts:
# part = part.headers.decode('utf-8')
if part.headers.get(b'Content-Type') == b'image/jpeg':
print(f' got an image')
img = part.content
print(f"{type(img)}")
print(f"{len(img)=}")
elif part.headers.get(b'Content-Type') == b'application/json':
print(f"got json data")
data = part.content.decode('utf-8')
print(data)
np_img = np.asarray(bytearray(img), dtype=np.uint8)
print(f"img after np,asarray(bytearray()) -> {type(np_img) = }")
print(f"{len(np_img)=}")
try:
new_img = cv2.imdecode(img, cv2.IMREAD_UNCHANGED)
except Exception as exc:
print(f"EXCEPTION while cv2.imdecode")
print(exc)
else:
print(f"img after cv2.imdecode -> {type(new_img) = }")
if new_img is not None:
if options.get("resize", 'no') != "no":
new_img = resize_image(img, options.get("resize"))
data["matched_data"]["image"] = new_img
else:
print(f"exiting due to image error")
g.logger.log_close(exit=1)
g.logger.log_close(exit=1)
exit(1)
return data
finally:
if do_raise:
raise ValueError('MLAPI remote detection error!')
Here is some of the code in the flask app that handles grabbing the image from the API and encoding it to be passed around to the ML model pipelines. This code works as expected.
r = response
img = np.asarray(bytearray(response.content), dtype="uint8")
img = cv2.imdecode(img, cv2.IMREAD_COLOR) # RGB ?
self.orig_h_w = img.shape[:2]
return img
So now there is a variable that contains img that is decoded into a jpg by cv2.imdecode. This image format (numpy.ndarray) can then be passed to the opencv DNN modules or be passed off to the pycoral models for TPU inference. This is the image I want to send back to the client. Here is the code I am using to accomplish that.
img = matched_data['image'].tobytes()
# Remove the numpy.ndarray formatted image from matched_data because it is not JSON serializable
matched_data['image'] = None
# Construct a multipart response that contains the detection data and the image
success = False
if matched_data["frame_id"]:
success = True
resp_json = {
'success': success,
'matched_data': matched_data,
'all_matches': all_matches,
}
from requests_toolbelt import MultipartEncoder
multipart_encoded_data = MultipartEncoder(
fields={
'json': (None, json.dumps(resp_json), 'application/json'),
'image': (f"event-{g.eid}-frame-{matched_data['frame_id']}.jpg", img, 'image/jpeg')
}
)
response = Response(multipart_encoded_data.to_string(), mimetype=multipart_encoded_data.content_type)
if success:
g.logger.info(
f"{lp} returning matched detection -> {matched_data}",
)
g.logger.debug(
f"{lp} returning all detections -> {all_matches}")
else:
g.logger.info(
f"{lp} no detections to return"
)
return response
Now on the client side separate the JSON and image and convert the image into a usable format ->
do_raise = False
data: Optional[dict] = None
multipart_data: decoder.MultipartDecoder = decoder.MultipartDecoder.from_response(r)
part: decoder.MultipartDecoder.from_response
img: Optional[bytes] = None
for part in multipart_data.parts:
# part = part.headers.decode('utf-8')
if part.headers.get(b'Content-Type') == b'image/jpeg':
print(f' got an image')
img = part.content
print(f"{type(img)}")
print(f"{len(img)=}")
elif part.headers.get(b'Content-Type') == b'application/json':
print(f"got json data")
data = part.content.decode('utf-8')
print(data)
np_img = np.asarray(bytearray(img), dtype=np.uint8)
print(f"img after np,asarray(bytearray()) -> {type(np_img) = }")
print(f"{len(np_img)=}")
try:
new_img = cv2.imdecode(img, cv2.IMREAD_UNCHANGED)
except Exception as exc:
print(f"EXCEPTION while cv2.imdecode")
print(exc)
else:
print(f"img after cv2.imdecode -> {type(new_img) = }")
if new_img is not None:
if options.get("resize", 'no') != "no":
new_img = resize_image(img, options.get("resize"))
data["matched_data"]["image"] = new_img
else:
print(f"exiting due to image error")
g.logger.log_close(exit=1)
g.logger.log_close(exit=1)
exit(1)
return data
The error I receive is silent it just returns 'None' ->
# Grabbing image using an http request and converting into a jpeg
11/07/21 20:44:30.623202 zm_mlapi[37535] DBG1 Media:659 ['std.out' --> image from ZM API as response.content - type(img) = <class 'bytes'> - len(img) = 205125]
11/07/21 20:44:30.627857 zm_mlapi[37535] DBG1 Media:661 ['std.out' --> after np.asarray(bytearray(img), np.uint8) - type(img) = <class 'numpy.ndarray'> - len(img) = 205125]
11/07/21 20:44:30.658582 zm_mlapi[37535] DBG1 Media:663 ['std.out' --> after cv2.imdecode(img, cv2.IMREAD_COLOR) - type(img) = <class 'numpy.ndarray'> - len(img) = 1080]
11/07/21 20:44:30.67595 zm_mlapi[37535] DBG2 pyzm_utils:386 [resize:img: success using resize=800.0 - original dimensions: 1920*1080 - resized dimensions: 450*800]
11/07/21 20:44:30.678568 zm_mlapi[37535] DBG1 Media:681 ['std.out' --> after resize - type(img) = <class 'numpy.ndarray'> - len(img) = 450]
# returned image to the class that requested it (ML Pipeline)
11/07/21 20:44:30.687835 zm_mlapi[37535] DBG1 detect_sequence:1048 ['std.out' --> DETECT STREAM: FRAME RETURNED FROM MEDIA CLASS --> type(frame) = <class 'numpy.ndarray'> - len(frame) = 450]
11/07/21 20:44:33.582062 zm_mlapi[37535] DBG1 detect_sequence:1656 ['std.out' --> before returning matched data - type(matched_data['image']) = <class 'numpy.ndarray'> - len(matched_data['image']) = 450]
# Return image to the flask app, now the flask app has to construct a response with JSON and the image
11/07/21 20:44:33.588139 zm_mlapi[37535] DBG1 mlapi:587 ['std.out' --> type(matched_data['image']) = <class 'numpy.ndarray'> - len(matched_data['image']) = 450]
11/07/21 20:44:33.591981 zm_mlapi[37535] DBG1 mlapi:590 ['std.out' --> before converting using .tobytes() - type(img) = <class 'numpy.ndarray'> - len(img) = 450]
11/07/21 20:44:33.596642 zm_mlapi[37535] DBG1 mlapi:594 ['std.out' --> after converting using .tobytes() - type(img) = <class 'bytes'> - len(img) = 1080000]
11/07/21 20:44:33.611218 zm_mlapi[37535] DBG1 mlapi:611 ['std.out' --> multipart MIME TYPE -> multipart/form-data; boundary=e7f7b825a51d4184ad7f12e7bbc6f411]
# flask app returns the response to the client
11/07/21 21:00:58.393864 zmesdetect_m4[102768] DBG1 zm_detect:418 ['std.out' --> got json data]
11/07/21 21:00:58.395459 zmesdetect_m4[102768] DBG1 zm_detect:414 ['std.out' --> got an image with Content-Type - b'application/octet']
11/07/21 21:00:58.396815 zmesdetect_m4[102768] DBG1 zm_detect:422 ['std.out' --> success = True]
11/07/21 21:00:58.398169 zmesdetect_m4[102768] DBG1 zm_detect:423 ['std.out' --> img - type(img) = <class 'bytes'> - len(img) = 1080000]
11/07/21 21:00:58.39958 zmesdetect_m4[102768] DBG1 zm_detect:424 ['std.out' --> img[:50] = b'\\gu\\gu\\gu]hv^iw_jx`kyalzgr\x80kv\x84it\x82it\x82it\x82it\x82it\x82it\x82ju']
11/07/21 21:00:58.401012 zmesdetect_m4[102768] DBG1 zm_detect:426 ['std.out' --> img after np.frombuffer(img, dtype=np.uint8) -> type(np_img) = <class 'numpy.ndarray'>]
11/07/21 21:00:58.402911 zmesdetect_m4[102768] DBG1 zm_detect:430 ['std.out' --> img after np_img.copy() -> type(np_img) = <class 'numpy.ndarray'>]
11/07/21 21:00:58.404296 zmesdetect_m4[102768] DBG1 zm_detect:432 ['std.out' --> len(np_img)=1080000]
11/07/21 21:00:58.405619 zmesdetect_m4[102768] DBG1 zm_detect:433 ['std.out' --> attempting to decode numpy array into a jpeg]
11/07/21 21:00:58.407144 zmesdetect_m4[102768] DBG1 zm_detect:442 ['std.out' --> img after cv2.imdecode -> type(new_img) = <class 'NoneType'>]
11/07/21 21:00:58.408474 zmesdetect_m4[102768] DBG1 zm_detect:448 ['std.out' --> exiting due to image error]
Any insight would be appreciated! I am new to SO so hopefully this is a proper question.
