Morningstar.com login using Python Requests

Question

I am trying to log in to my Morningstar.com premium account using the requests module in python as below. The post command runs through with status 200 but does not actually log me in.

(When I download the balance sheet, I only receive the 5 year (non-premium) version instead of the requested 10 year (premium) version. This indicates that my login script fails, since the 5 year data is available without login. The balance sheet URL works correctly right when logging in manually in the browser.)

Does anybody know how to correctly set up the login script?

It seems very straight forward but I have tried the whole day using different forms of the payload/ headers etc. and can't find the right way... Also, I am confused since I cannot find the Form Data information when inspecting the login page.

import csv
import requests

urlLogin = 'http://members.morningstar.com/memberservice/login.aspx'
urlBalanceSheet = 'http://financials.morningstar.com/ajax/ReportProcess4CSV.html?&t=XNYS:F&region=usa&culture=en-US&cur=&reportType=bs&period=12&dataType=A&order=desc&columnYear=10&rounding=1&view=raw&r=149906&denominatorView=raw&number=1'

payload = {
    "uEmail": "<userEmail>",
    "uPassword": "<userPW>",
    "remember_me": "on",
    "login": "Sign In"
}

with requests.Session() as s:
    p = s.post(urlLogin, data = payload)
    print(p.status_code)

    download = s.get(urlBalanceSheet)

Answer 1

There are few things you can do to automate downloading from morningstar

pip install selenium http://selenium-python.readthedocs.io/installation.html

install firefox, find out where your profile is here is an resource http://toolsqa.com/selenium-webdriver/custom-firefox-profile/

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import time
import requests
from xml.etree import cElementTree as ET
import csv
from selenium.webdriver.common.action_chains import ActionChains


def timeme(method):
    def wrapper(*args, **kw):
        startTime = int(round(time.time() * 1000))
        result = method(*args, **kw)
        endTime = int(round(time.time() * 1000))

        print(endTime - startTime, 'ms')
        return result

    return wrapper

class Driver():
    def __init__(self,profile, diver_path, url):
        self.profile = profile
        self.driver_path = diver_path
        self.url = url

    def start_driver(self):
        user_profile = webdriver.FirefoxProfile(self.profile)
        user_profile.set_preference("browser.helperApps.neverAsk.saveToDisk", 'text/csv')

        driver = webdriver.Firefox(executable_path=self.driver_path, firefox_profile=user_profile)
        driver.get(self.url)

        return driver

    def shutdown(self,driver):

        driver.quit()


@timeme
def login(driver, email = '', password = ''):

    wait_time = 1

    try:

        email_input = WebDriverWait(driver,wait_time).until(
        EC.presence_of_all_elements_located((By.XPATH,'//*[@id="uim-uEmail-input"]')))


        email_input = driver.find_element_by_xpath('//*[@id="uim-uEmail-input"]').send_keys(email)

        time.sleep(5) # wait time to see if you have input remove later
        pwd_input = driver.find_element_by_xpath('//*[@id="uim-uPassword-input"]').send_keys(password)
        time.sleep(5)
        sign_in = driver.find_element_by_xpath('//*[@id="uim-login-submit"]').click()



        title = driver.title
        driver.execute_script("window.open('http://financials.morningstar.com/ajax/ReportProcess4CSV.html?&t=XNYS:F&region=usa&culture=en-US&cur=&reportType=bs&period=12&dataType=A&order=desc&columnYear=10&rounding=1&view=raw&r=149906&denominatorView=raw&number=1','new_window');") 
        time.sleep(1)

        return 0
    except Exception as e:

        return None

@timeme
def main():
    # i am using on my mac, if you are using windows change paths accordingly
    Mozilla  = Driver(profile = '/Users/yourname/Library/Application Support/Firefox/Profiles/xxxxxxxxxxxx.default',
           diver_path='/usr/local/bin/geckodriver', # path to firefox driver
           url='https://www.morningstar.com/members/login.html?vurl=')

    driver = Mozilla.start_driver()
    download = login(driver, password='', email='')
    if download ==0:
       time.sleep(10) # let browser to download csv

       Mozilla.shutdown(driver) # shutdown 

main()