用硒4刮擦

Question

这是我要抓取的 HTML 代码：

<div class="vbseo_liked">
<a href="http://www.jamiiforums.com/member.php?u=8355" rel="nofollow">Nyaralego</a>
,
<a href="http://www.jamiiforums.com/member.php?u=8870" rel="nofollow">Sikonge</a>
,
<a href="http://www.jamiiforums.com/member.php?u=8979" rel="nofollow">Ab-Titchaz</a>
and
<a onclick="return vbseoui.others_click(this)" href="http://www.jamiiforums.com/kenyan-news/225589-kenyan-and-tanzanian-surburbs.html#">11 others</a>
like this.
</div>

这是我目前的代码：

from selenium import webdriver
from selenium.webdriver.common.keys import Keys


path_to_chromedriver = '/Users/myMac/Desktop/chromedriver' # change path as needed
#browser = webdriver.Chrome(executable_path = path_to_chromedriver)
browser= webdriver.PhantomJS()
browser.set_window_size(1120, 550)
#browser.get("http://www.jamiiforums.com/login.php?do=logout&logouthash=1427579705-9d07da3339a5f07bb3a4c9d1001b75dc0431a138")
browser.get("http://www.jamiiforums.com/kenyan-news/225589-kenyan-and-tanzanian-surburbs.html")
username = browser.find_element_by_id("navbar_username")
password = browser.find_element_by_name("vb_login_password_hint")

username.send_keys("")
password.send_keys("")

browser.find_element_by_class_name("loginbutton").click()
link = browser.find_element_by_xpath('//div[@class="vbseo_liked"]/a[contains(@onclick, "return vbseoui.others_click(this)")]')
link.click()

print 'succes!!'
browser.close()

我希望能够点击链接：

<a onclick="return vbseoui.others_click(this)" href="http://www.jamiiforums.com/kenyan-news/225589-kenyan-and-tanzanian-surburbs.html#">11 others</a>

然后在点击链接后在屏幕上打印出结果。

Answer 1

登录后发生重定向。在获取页面源之前，您需要等待重定向发生。这对我有用：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

browser = webdriver.PhantomJS()
browser.set_window_size(1120, 550)
browser.get("http://www.jamiiforums.com/kenyan-news/225589-kenyan-and-tanzanian-surburbs.html")

username = browser.find_element_by_id("navbar_username")
password = browser.find_element_by_name("vb_login_password_hint")

username.send_keys("username")
password.send_keys("password")

browser.find_element_by_class_name("loginbutton").click()

wait = WebDriverWait(browser, 10)
wait.until(EC.visibility_of_element_located((By.XPATH, '//h2[contains(., "Redirecting")]')))
wait.until(EC.title_contains('Kenyan & Tanzanian'))

print(browser.page_source)

print 'success!!'
browser.close()