Connection Aborted Error when scraping with requests module

Question

I'm using the request module to do some web scraping in python, but everytime i send the requests with headers and proxies, i get a connection aborted error, even though i've been told that it would solve the problem if i did put the header.

This is my full code:

import requests 
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import pickle 


session = requests.Session()
unique = {}
retry = Retry(connect=3,backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://',adapter)
headers= {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0'}
proxies1 = {"http":"http://23.238.33.186:80"}


lstofwords = pickle.load(f)
dic={}
first = int(len(lstofwords)/3)
for x in range(0,first):
    r = session.get(f'https://www.merriam-webster.com/dictionary/{lstofwords[x]}/',headers=headers,proxies=proxies1)
    soup = BeautifulSoup(r.content,'html.parser')
    descriptions = soup.find_all('span', class_='unText') 
    descs = [i.text for i in descriptions]
    dic[lstofwords[x]]= descs

for x in range(first,second):
    r = session.get(f'https://www.merriam-webster.com/dictionary/{lstofwords[x]}/',headers=headers,proxies=proxies2)
    soup = BeautifulSoup(r.content,'html.parser')
    descriptions = soup.find_all('span', class_='unText') 
    descs = [i.text for i in descriptions]
    dic[lstofwords[x]]= descs

for x in range(second,third):
    r = session.get(f'https://www.merriam-webster.com/dictionary/{lstofwords[x]}/',headers=headers,proxies=proxies3)
    soup = BeautifulSoup(r.content,'html.parser')
    descriptions = soup.find_all('span', class_='unText') 
    descs = [i.text for i in descriptions]
    dic[lstofwords[x]]= descs

with open('dictionary.pkl','wb') as f:
    pickle.dump(dic, f)
print("Success")

Below is the error message I'm getting:


Traceback (most recent call last):
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 449, in _make_request
    six.raise_from(e, None)
  File "<string>", line 3, in raise_from
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 444, in _make_request
    httplib_response = conn.getresponse()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1374, in getresponse
    response.begin()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 318, in begin
    version, status, reason = self._read_status()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 287, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\adapters.py", line 440, in send
    resp = conn.urlopen(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 785, in urlopen
    retries = retries.increment(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\util\retry.py", line 550, in increment
    raise six.reraise(type(error), error, _stacktrace)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\packages\six.py", line 769, in reraise
    raise value.with_traceback(tb)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 449, in _make_request
    six.raise_from(e, None)
  File "<string>", line 3, in raise_from
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 444, in _make_request
    httplib_response = conn.getresponse()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1374, in getresponse
    response.begin()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 318, in begin
    version, status, reason = self._read_status()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.1520.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 287, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\ehabb\Downloads\research.py", line 53, in <module>
    r = session.get(f'https://www.merriam-webster.com/dictionary/{lstofwords[x]}/',headers=headers,proxies=proxies1)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 542, in get
    return self.request('GET', url, **kwargs)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 529, in request
    resp = self.send(prep, **send_kwargs)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 667, in send
    history = [resp for resp in gen]
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 667, in <listcomp>
    history = [resp for resp in gen]
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 237, in resolve_redirects
    resp = self.send(
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 645, in send
    r = adapter.send(request, **kwargs)
  File "C:\Users\ehabb\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\adapters.py", line 501, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))

Does this answer your question? [Python HTTP Server/Client: Remote end closed connection without response error](https://stackoverflow.com/questions/48105448/python-http-server-client-remote-end-closed-connection-without-response-error) — Deera Wijesundara, Jun 28 '22 at 05:52

score 0 · Accepted Answer · answered Jun 28 '22 at 07:48

I guess the issue can be with the protocol of the proxy that you use. The proxy is for http protocol, while you're querying https resource. Try to add a line with an https proxy (if it is available):

proxies1 = {
    "http": "http://23.238.33.186:80",
    "https": "https://23.238.33.186:443"
}

Connection Aborted Error when scraping with requests module

1 Answers1