Unpickle error sending over Internet but not locally

Question

I'm trying to run a server for a monopoly game I made.

Running it locally works fine but over the internet it breaks because of an unpickling error. Clientside:

File "/home/john/PycharmProjects/Monopoly_py/Client.py", line 6, in <module>
    message, is_req = n.recv()
  File "/home/john/PycharmProjects/Monopoly_py/Network.py", line 24, in recv
    header = pickle.loads(header)
_pickle.UnpicklingError: invalid load key, ' '.

While on the Serverside it moves on and tries to recieve data in a line later in the script.

File "Server.py", line 152, in <module>
    command = s.request("str", f"{user.name}>", user)
  File "/monopoly/Server_Network.py", line 60, in request
    reply = self.recv(conn)
  File "/monopoly/Server_Network.py", line 15, in recv
    header_size = conn.recv(2)

this happens after sending updated board in a loop like this

for user in users:
    for i in range(40):
        network.send("str", f"{about: 170chars of text}", user)

My network script looks like this

class Network:
    def __init__(self):
        self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.host = "ip to be filled"
        self.port = "port to be filled"
        self.addr = (self.host, self.port)
        self.client.bind(self.addr)

    def recv(self, conn):
        header_size = conn.recv(2)
        header_size = int.from_bytes(header_size, "little")
        header = conn.recv(header_size)
        if not header:
            return
        header = pickle.loads(header)
        data = conn.recv(header["message_length"])
        if header["type"] == "obj":
            data = pickle.dumps(data)
        elif header["type"] == "str":
            data = data.decode("utf-8")
        return data

    def send(self, data_type, data, user):
        conn = user.conn
        if data_type == "obj":
            array = pickle.dumps(data)
        elif data_type == "str":
            array = str.encode(data)
        else:
            return
        header = {"type": data_type, "message_length": len(array), "request": False}
        header_array = pickle.dumps(header)
        header_length = len(header_array)
        header_length = int.to_bytes(header_length, 2, "little")
        conn.sendall(header_length)
        conn.sendall(header_array)
        conn.sendall(array)

    def request(self, data_type, data, user, force_strings=True):
        conn = user.conn
        if data_type == "obj":
            array = pickle.dumps(data)
        elif data_type == "str":
            array = str.encode(data)
        else:
            return
        header = {"type": data_type, "message_length": len(array), "request": True}
        header_array = pickle.dumps(header)
        header_length = len(header_array)
        header_length = int.to_bytes(header_length, 2, "little")
        conn.sendall(header_length)
        conn.sendall(header_array)
        conn.sendall(array)
        if force_strings:
            reply = self.recv(conn)
            if type(reply) != str:
                print(f"{user.name} SEEMS TO USE A MODIFIED CLIENT")
            reply = f"{reply}"
        else:
            reply = self.recv(conn)
        return reply

while the client Network script looks like this

import socket
import pickle

class Network:
    def __init__(self):
        self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server = "ip to be filled"
        self.port = "port to be filled"
        self.addr = (self.server, self.port)
        self.connect()

    def connect(self):
        try:
            self.client.connect(self.addr)
            print(self.recv())
            print("======")
        except:
            pass

    def recv(self):
        header_size = self.client.recv(2)
        header_size = int.from_bytes(header_size, "little")
        header = self.client.recv(header_size)
        header = pickle.loads(header)
        data = self.client.recv(header["message_length"])
        if header["type"] == "obj":
            data = pickle.dumps(data)
        elif header["type"] == "str":
            data = data.decode("utf-8")
        return data, header["request"]



    def send(self, type, data):
        if type == "obj":
            array = pickle.dumps(data)
        elif type == "str":
            array = str.encode(data)
        else:
            return
        header = {"type": type, "message_length": len(array)}
        header_array = pickle.dumps(header)
        header_length = len(header_array)
        header_length = int.to_bytes(header_length, 2, "little")
        self.client.sendall(header_length)
        self.client.sendall(header_array)
        self.client.sendall(array)

When i run it slower aka 50ms second delay it works better but still breaks after some time.

Please read [this answer](https://stackoverflow.com/a/43420503/238704) to understand why your protocol is buggy. Note that this link is actually right in the description for the [sockets tag](https://stackoverflow.com/tags/sockets/info). — President James K. Polk, Dec 24 '20 at 21:30
@PresidentJamesK.Polk I read the full answer but still don't understand why this would make the protocoll buggy. I'm aware that the messages come in after one another but without terminator. This is why i send the header. I've noticed though that the amount of bytes specified isn't equal to the bytes recieved. — John Janzen, Dec 24 '20 at 21:59
@PresidentJamesK.Polk also i do not end my messages by checking if the other party has disconnected or not, but by not recieving bytes more than the "message_length" attribute specifies — John Janzen, Dec 24 '20 at 22:02

score 0 · Answer 1 · answered Dec 25 '20 at 12:59

Yeah, so the answer was to implement a "receive all" function that receives an exact number on bytes

data = self.client.recv(header["message_length"])
while header["message_length"] > len(data):
    data += self.client.recv(header["message_length"]-len(data))

For everything that is being send

Unpickle error sending over Internet but not locally

1 Answers1