python - 使用 Python 请求登录 Morningstar.com

Question

我正尝试使用 python 中的请求模块 登录我的 Morningstar.com 高级帐户，如下所示。 post 命令以状态 200 运行，但实际上并没有让我登录。

(当我下载 Assets 负债表时，我只收到 5 年(非高级)版本，而不是请求的 10 年(高级)版本。这表明我的登录脚本失败，因为 5 年数据无需登录即可使用。在浏览器中手动登录时， Assets 负债表 URL 可以正常工作。)

有人知道如何正确设置登录脚本吗？

这看起来很简单，但我一整天都在尝试使用不同形式的有效负载/ header 等，但找不到正确的方法……另外，我很困惑，因为当我找不到表单数据信息时检查登录页面。

import csv
import requests

urlLogin = 'http://members.morningstar.com/memberservice/login.aspx'
urlBalanceSheet = 'http://financials.morningstar.com/ajax/ReportProcess4CSV.html?&t=XNYS:F&region=usa&culture=en-US&cur=&reportType=bs&period=12&dataType=A&order=desc&columnYear=10&rounding=1&view=raw&r=149906&denominatorView=raw&number=1'

payload = {
    "uEmail": "<userEmail>",
    "uPassword": "<userPW>",
    "remember_me": "on",
    "login": "Sign In"
}

with requests.Session() as s:
    p = s.post(urlLogin, data = payload)
    print(p.status_code)

    download = s.get(urlBalanceSheet)

Answer 1

您可以做一些事情来自动从 morningstar 下载

pip 安装 Selenium http://selenium-python.readthedocs.io/installation.html

安装 firefox，找出你的个人资料在哪里 这是一个资源 http://toolsqa.com/selenium-webdriver/custom-firefox-profile/

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import time
import requests
from xml.etree import cElementTree as ET
import csv
from selenium.webdriver.common.action_chains import ActionChains


def timeme(method):
    def wrapper(*args, **kw):
        startTime = int(round(time.time() * 1000))
        result = method(*args, **kw)
        endTime = int(round(time.time() * 1000))

        print(endTime - startTime, 'ms')
        return result

    return wrapper

class Driver():
    def __init__(self,profile, diver_path, url):
        self.profile = profile
        self.driver_path = diver_path
        self.url = url

    def start_driver(self):
        user_profile = webdriver.FirefoxProfile(self.profile)
        user_profile.set_preference("browser.helperApps.neverAsk.saveToDisk", 'text/csv')

        driver = webdriver.Firefox(executable_path=self.driver_path, firefox_profile=user_profile)
        driver.get(self.url)

        return driver

    def shutdown(self,driver):

        driver.quit()


@timeme
def login(driver, email = '', password = ''):

    wait_time = 1

    try:

        email_input = WebDriverWait(driver,wait_time).until(
        EC.presence_of_all_elements_located((By.XPATH,'//*[@id="uim-uEmail-input"]')))


        email_input = driver.find_element_by_xpath('//*[@id="uim-uEmail-input"]').send_keys(email)

        time.sleep(5) # wait time to see if you have input remove later
        pwd_input = driver.find_element_by_xpath('//*[@id="uim-uPassword-input"]').send_keys(password)
        time.sleep(5)
        sign_in = driver.find_element_by_xpath('//*[@id="uim-login-submit"]').click()



        title = driver.title
        driver.execute_script("window.open('http://financials.morningstar.com/ajax/ReportProcess4CSV.html?&t=XNYS:F&region=usa&culture=en-US&cur=&reportType=bs&period=12&dataType=A&order=desc&columnYear=10&rounding=1&view=raw&r=149906&denominatorView=raw&number=1','new_window');") 
        time.sleep(1)

        return 0
    except Exception as e:

        return None

@timeme
def main():
    # i am using on my mac, if you are using windows change paths accordingly
    Mozilla  = Driver(profile = '/Users/yourname/Library/Application Support/Firefox/Profiles/xxxxxxxxxxxx.default',
           diver_path='/usr/local/bin/geckodriver', # path to firefox driver
           url='https://www.morningstar.com/members/login.html?vurl=')

    driver = Mozilla.start_driver()
    download = login(driver, password='', email='')
    if download ==0:
       time.sleep(10) # let browser to download csv

       Mozilla.shutdown(driver) # shutdown 

main()