Send bytes in json format to the server in socket programming python

Question

I wrote a socket

What I want to do is select a file

After that, read the file

and after that convert the readed file to a list and building a list of bytes

And every time I send one of the indexes of this list to the server separately in json format

and Load the json on the server side and print it

everything all right

It means that the file is read correctly

properly converted to a list of bytes

and it is correctly sent to the server in json format

But on the server side when I want to get it out of json format

i get this error ->

{'type': 'fileUpload', 'file': 'iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAACXBIWXMAAA7EAAAOxAGVKw4bAAATPElEQVR42s1be2xT1R//3Hv7Wtm6dXNjZWxswgabDsabgThIjAMVBKe44BOjRjBEUYnySAQfBBPBVwJBxWCMiqIoBAQHSoLKBBcCDDYeY2wdm92rXdv1dV/f3x+kl7a7bccYv/hNbnp7envu+Z7zPd/v53zO9zB0TQAADMMgeB/8DgChvwe/q92HSrTy0N8i6+np6QHDMOjo6IAkSRg9ejRYlgURQRAE/PPPPxg5ciQyMzP7VX+0d4XpFNS+u7sbAJCWlgYigtfrxZAhQ/r8IbSj1CqO1qDITlVrsCzLAABZlkFE0Gq1yjOSJKGzsxPJyckwGAzqykRRNFZ7NcGHAoEAJElCWlqa8kI15dQqH4hIkgRJkiDLMliWhU6nA8uyUS2N4zhl5COVjmxD6P/UPsN+p9DhiaJYNJONvJckCe3t7ZBlGSaTCQaDARzHQZZldHd3w+12w2q1orq6Gr29vfB4PHC73TCZTHj44YcxdepUsCwLnueRkJCgmP/NTLdY7QcARpZl6q9Jhz6jZg29vb147rnncPjwYZjNZlgsFuTk5MDhcODkyZNwOp0IBAKKqYeKwWDAQw89hKFDh6KmpgZlZWV48sknMWrUqD7vVBmzfvuryGmsiTSRWBKtg2RZhtVqRV1dHXieR3d3N7q7u9HY2IiamhqwLIsFCxZAlmV89913qnX7/X58++23yqg7HA40NTVh+PDhKC8vx6hRo5Camgqj0RjVCgKBAHQ6HQRBgF6vj9pJoR2jiWc28aZGU1MTvvrqK9TV1cFsNkMQBBgMBoiiiFmzZiE/Px88zyMjIwP//POP6ujpdDoYDAbk5eXh6tWrcLlcEAQBL774Ik6ePInXX38dDMNg69atKCkpUVWEYRhoNBowDAOO42JGqzA/Qv0QWZb7lEmSRDU1NTR79myaMmUKVVVV0Z49e+iJJ56glJQUAkAcxxHHcQQg7GIYRrlnWZays7NJr9fTkiVLqKSkhMaMGUNFRUW0cuVKmjZtGqWkpBDDMPTII4/QqVOniOf5G2qzWvuDguCPkX8IXoFAgCKfkWWZqquracGCBfTll1+S1+ul48eP04QJEwgAaTSaPkoDoJycHCorK6MHHniAWJYlAGSxWOi+++6j4uJiWrRoERUWFtI777xDSUlJlJ2dTUOHDqXExESljvT0dNqxY4dqeyPv1Tol8jlWzQeEmorH41Gdr7t37wbP87jnnntw9OhRrFq1CmfPngUAiKKoGqK0Wi1mzJiBuXPnwmQyISsrC2VlZejq6sKiRYtQWVmJlA=='}
Exception in thread Thread-1:
Traceback (most recent call last):
  File "C:\Users\PC\AppData\Local\Programs\Python\Python39\lib\threading.py", line 980, in _bootstrap_inner
    self.run()
  File "C:\Users\PC\AppData\Local\Programs\Python\Python39\lib\threading.py", line 917, in run
    self._target(*self._args, **self._kwargs)
  File "C:\Users\PC\Desktop\New folder (5)\server.py", line 11, in listen_for_messages
    message = json.loads(message)
  File "C:\Users\PC\AppData\Local\Programs\Python\Python39\lib\json\__init__.py", line 346, in loads
    return _default_decoder.decode(s)
  File "C:\Users\PC\AppData\Local\Programs\Python\Python39\lib\json\decoder.py", line 340, in decode
    raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 1 column 1403 (char 1402)

client.py

from PyQt6.QtWidgets import *
from PyQt6.QtGui import *
from PyQt6.QtCore import *
import socket
import threading
import json
import os
import base64


class Window(QMainWindow):
    def __init__(self):
        super().__init__()
        self.show()

        self.fileBtn = QPushButton('file upload')
        self.fileBtn.clicked.connect(self.fileUpload)
        mainWidget = QWidget()
        layout = QVBoxLayout()
        mainWidget.setLayout(layout)
        layout.addWidget(self.fileBtn)
        self.setCentralWidget(mainWidget)

        self.connect()

    def connect(self):
        self.HOST = '127.0.0.1'
        self.PORT = 1234
        self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.client.connect((self.HOST, self.PORT))

    def fileUpload(self):
        fileName, ok = QFileDialog.getOpenFileName(self)
        if ok:
            file = open(fileName , 'rb')
            file = file.read()
            _range = 1024
            def reader(data):
                for i in range(0, len(data), _range):
                    yield data[i:i+_range]
            lst = list(reader(file))

            for elem in lst:
                dic = json.dumps({
                    "type":"fileUpload",
                    "file":base64.b64encode(elem).decode('utf-8'),
                })
                self.client.sendall(bytes(dic,encoding="utf-8"))      

app = QApplication([])
win = Window()
app.exec()

server.py

import socket
import threading
import json
LISTENER_LIMIT = 5
HOST = '127.0.0.1'
PORT = 1234

def listen_for_messages(client):
    while True:
        message = client.recv(2048).decode('utf-8')
        message = json.loads(message)
        print(message)

def main():

    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    try:
        server.bind((HOST, PORT))
    except:
        print(f"Unable to bind to host {HOST} and port {PORT}")
    server.listen(LISTENER_LIMIT)
    while 1:
        client,address = server.accept()
        threading.Thread(target=listen_for_messages, args=(client, )).start()

if __name__ == '__main__':
    main()

The server is reading the socket in 2048-byte messages. There's nothing in the client that ensures that each message is a single JSON object, so how do you expect `json.loads(message)` to work properly? — Barmar, Jun 29 '23 at 15:16
I think you're making the common assumption that each call to `send()` in the sender will correspond to a call to `recv()` in the recevier. This is true for datagram transport, but not stream. SOCK_STREAM doesn't maintain message boundaries. — Barmar, Jun 29 '23 at 15:21
Does this answer your question? [Python socket not receiving without sending](https://stackoverflow.com/questions/43420075/python-socket-not-receiving-without-sending) — President James K. Polk, Jun 29 '23 at 15:24
Don't split everything into separate messages. Make one big JSON object, send it with one `sendall()`, close the connection, then receive it with a single `read()`. This will read until EOF. — Barmar, Jun 29 '23 at 17:11

score 1 · Accepted Answer · answered Jul 19 '23 at 17:04

TCP is a streaming protocol. There are no message boundaries. It is up to you to read the byte stream and parse it for messages.

One easy method is to make a "message" be a newline-terminated string. socket has a .makefile method that wraps the TCP stream in a file-like object. This object has a .readline method. Send each JSON as a newline-terminated line.

Example:

server.py

import socket
import threading
import json

def listen_for_messages(client, address):
    with client.makefile('rb') as rfile:
        while True:
            line = rfile.readline()
            if not line: break
            message = json.loads(line)
            print(f'{address}: {message}')

server = socket.socket()
server.bind(('', 1234))
server.listen()
while True:
    client, address = server.accept()
    threading.Thread(target=listen_for_messages, args=(client, address)).start()

client.py

import socket
import json

with socket.socket() as s:
    s.connect(('localhost', 1234))
    lst = ['abc', 'def', 'ghi']
    for elem in lst:
        dic = json.dumps({'type': 'fileUpload', 'file': elem}) + '\n'
        s.sendall(dic.encode())

Start server.py and run client.py a few times.

Output example:

('127.0.0.1', 21062): {'type': 'fileUpload', 'file': 'abc'}
('127.0.0.1', 21062): {'type': 'fileUpload', 'file': 'def'}
('127.0.0.1', 21062): {'type': 'fileUpload', 'file': 'ghi'}
('127.0.0.1', 21065): {'type': 'fileUpload', 'file': 'abc'}
('127.0.0.1', 21065): {'type': 'fileUpload', 'file': 'def'}
('127.0.0.1', 21065): {'type': 'fileUpload', 'file': 'ghi'}
('127.0.0.1', 21278): {'type': 'fileUpload', 'file': 'abc'}
('127.0.0.1', 21278): {'type': 'fileUpload', 'file': 'def'}
('127.0.0.1', 21278): {'type': 'fileUpload', 'file': 'ghi'}

Send bytes in json format to the server in socket programming python

1 Answers1