【Python爬虫】之爬取页面内容、图片以及用selenium爬取

下面不做过多文字描述：

首先、安装必要的库

# 安装BeautifulSoup
pip install beautifulsoup4

# 安装requests
pip install requests

 

其次、上代码！！！

①重定向网站爬虫h4文字

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from PIL import Image


# 重定向爬虫h4
url = "http://www.itest.info/courses"
soup = BeautifulSoup(requests.get(url).text,\'html.parser\')

for courses in soup.find_all(\'p\'):
    print(courses.text)
    print("\r")

②v2ex爬取标题

import requests
from bs4 import BeautifulSoup

# v2ex爬虫标题
url = "https://www.v2ex.com"
v2ex = BeautifulSoup(requests.get(url).text,\'html.parser\')

for span in v2ex.find_all(\'span\',class_=\'item_hot_topic_title\'):
    print(span.find(\'a\').text,span.find(\'a\')[\'href\'])

for title in v2ex.find_all("a",class_="topic-link"):
    print(title.text,url+title["href"])

③煎蛋爬虫图片

import requests
from bs4 import BeautifulSoup



headers = {
    \'user-agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36\'
}

def download_file(url):
    \'\'\'下载图片\'\'\'
    print(\'Downding %s\' %url)
    local_filename = url.split(\'/\')[-1]
    # 指定目录保存图片
    img_path = "/Users/zhangc/Desktop/GitTest/project_Buger_2/Python爬虫/img/" + local_filename
    print(local_filename)
    r = requests.get(url, stream=True, headers=headers)
    with open(img_path, \'wb\') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
                f.flush()
    return img_path

url = \'http://jandan.net/drawings\'
soup = BeautifulSoup(requests.get(url, headers=headers).text, \'html.parser\')

def valid_img(src):
    \'\'\'判断地址符不符合关键字\'\'\'
    return src.endswith(\'jpg\') and \'.sinaimg.cn\' in src

for img in soup.find_all(\'img\', src=valid_img):
    src = img[\'src\']
    if not src.startswith(\'http\'):
        src = \'http:\' + src
    download_file(src)

④爬取知乎热门标题

import requests
from bs4 import BeautifulSoup

headers ={
    "user-agent":"user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
}
url = "https://www.zhihu.com/explore"
zhihu = BeautifulSoup(requests.get(url,headers=headers).text,"html.parser")
for title in zhihu.find_all(\'a\',class_="ExploreSpecialCard-contentTitle"):
    print(title.text)

⑤selenium爬虫知乎热门标题

import requests
from bs4 import BeautifulSoup


# selenium爬虫
url = "https://www.zhihu.com/explore"
driver = webdriver.Chrome("/Users/zhangc/Desktop/GitTest/project_Buger_2/poium测试库/tools/chromedriver")
driver.get(url)

info = driver.find_element(By.CSS_SELECTOR,"div.ExploreHomePage-specials")
for title in info.find_elements(By.CSS_SELECTOR,"div.ExploreHomePage-specialCard > div.ExploreSpecialCard-contentList > div.ExploreSpecialCard-contentItem > a.ExploreSpecialCard-contentTitle"):
    print(title.text,title.get_attribute(\'href\'))