Web-scraping Remax.com in python

Question

I am trying to follow the tutorial here to scrape data from Remax.com. At the moment I am just interested in getting the sqft of a particular home. Although I get this error:

Error during requests to https://www.remax.com/realestatehomesforsale/25-montage-way-laguna-beach-ca-92651-gid100012499996.html : HTTPSConnectionPool(host='www.remax.com', port=443): Max retries exceeded with url: /realestatehomesforsale/25-montage-way-laguna-beach-ca-92651-gid100012499996.html (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'ssl3_get_server_certificate', 'certificate verify failed')],)",),))
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-7-28b8e2248942> in <module>()
      1 raw_html = simple_get('https://www.remax.com/realestatehomesforsale/25-montage-way-laguna-beach-ca-92651-gid100012499996.html')
----> 2 html = BeautifulSoup(raw_html, 'html.parser')
      3 for i, li in enumerate(html.select('li')):
      4         print(i, li.text)

~\AppData\Local\Continuum\anaconda3\lib\site-packages\bs4\__init__.py in __init__(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, **kwargs)
    190         if hasattr(markup, 'read'):        # It's a file-type object.
    191             markup = markup.read()
--> 192         elif len(markup) <= 256 and (
    193                 (isinstance(markup, bytes) and not b'<' in markup)
    194                 or (isinstance(markup, str) and not '<' in markup)

TypeError: object of type 'NoneType' has no len()

Here is my entire code thus far:

from requests import get
from requests.exceptions import RequestException
from contextlib import closing
from bs4 import BeautifulSoup

def simple_get(url):
    """
    Attempts to get the content at `url` by making an HTTP GET request.
    If the content-type of response is some kind of HTML/XML, return the
    text content, otherwise return None.
    """
    try:
        with closing(get(url, stream=True)) as resp:
            if is_good_response(resp):
                return resp.content
            else:
                return None

    except RequestException as e:
        log_error('Error during requests to {0} : {1}'.format(url, str(e)))
        return None
def is_good_response(resp):
    """
    Returns True if the response seems to be HTML, False otherwise.
    """
    content_type = resp.headers['Content-Type'].lower()
    return (resp.status_code == 200 
            and content_type is not None 
            and content_type.find('html') > -1)

raw_html = simple_get('https://www.remax.com/realestatehomesforsale/25-montage-way-laguna-beach-ca-92651-gid100012499996.html')
html = BeautifulSoup(raw_html, 'html.parser')
for i, li in enumerate(html.select('li')):
        print(i, li.text)

I am pretty new to web scraping so I am not sure how to fix this. Any suggestions would be greatly appreciated.

Answer 1

Not sure about your question, but if all you're interested is the square footage of the house on that page, you can use

    import urllib
    from bs4 import BeautifulSoup

    url = 'https://www.remax.com/realestatehomesforsale/25-montage-way-laguna-beach-ca-92651-gid100012499996.html'

    hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
   'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
   'Accept-Encoding': 'none',
   'Accept-Language': 'en-US,en;q=0.8',
   'Connection': 'keep-alive'}

    request = urllib.request.Request(url, headers=hdr)
    html = urllib.request.urlopen(request).read()

    soup = BeautifulSoup(html,'html.parser')
    foot = soup.find('span', class_="listing-detail-sqft-val")
    foot.text.strip()

Output:

'7,604'

Answer 2

Your simple_get() function has been made to return None if the request fails. You should therefore test for this before using it. This could be done as follows:

from requests import get
from requests.exceptions import RequestException
from contextlib import closing
from bs4 import BeautifulSoup

def simple_get(url):
    """
    Attempts to get the content at `url` by making an HTTP GET request.
    If the content-type of response is some kind of HTML/XML, return the
    text content, otherwise return None.
    """

    try:
        with closing(get(url, stream=True)) as resp:
            if is_good_response(resp):
                return resp.content
            else:
                return None

    except RequestException as e:
        log_error('Error during requests to {0} : {1}'.format(url, str(e)))
        return None


def is_good_response(resp):
    """
    Returns True if the response seems to be HTML, False otherwise.
    """
    content_type = resp.headers['Content-Type'].lower()

    return (resp.status_code == 200 
            and content_type is not None 
            and content_type.find('html') > -1)


url = 'https://www.remax.com/realestatehomesforsale/25-montage-way-laguna-beach-ca-92651-gid100012499996.html'
raw_html = simple_get(url)

if raw_html:
    html = BeautifulSoup(raw_html, 'html.parser')

    for i, li in enumerate(html.select('li')):
            print(i, li.text)
else:
    print(f"get failed for '{url}'")

So to simplfy, the following would give you the same error message:

from bs4 import BeautifulSoup

html = BeautifulSoup(None, 'html.parser')