校花网视频爬取

# 简易版爬取校花网视频
import requests
import re
import hashlib
import time

movie_path = "C:\mp4"     #视频存储目录

def get_page(url):
    \'\'\'
    返回一个字符串的网页页面
    :param url: 
    :return: 
    \'\'\'
    try:
        response = requests.get(url)      # 请求传入的url
        if response.status_code == 200:   # 如果页面返回200：正常返回text字符串
            return response.text
    except Exception:
        pass

def parse_index(index_page):
    \'\'\'
    正则匹配到页面中的每个视频链接[],[],[],[],[],[],[]...
    :param index_page: 
    :return: 每次返回一个列表，也就是一个url
    \'\'\'
    urls = re.findall(\'class="items".*?href="(.*?)"\',index_page,re.S)   #查找匹配的url
    for url in urls:
        print(url)
        if not url.startswith("http"):
            \'\'\'
            因为某些链接不知只是域名后边的字符串，所以加判断，
            \'\'\'
            url = "http://www.xiaohuar.com"+url
        yield url

def parse_detail(detail_page):
    \'\'\'
    接收上面函数传入的url，正则匹配查到视频的url链接
    :param detail_page: 
    :return: 返回视频的url链接
    \'\'\'
    l = re.findall(\'id="media".*?src="(.*?)"\',detail_page,re.S)
    if l:
        movie_url = l[0]
        if movie_url.endswith("mp4"):
            yield movie_url


def get_movie(url):
    \'\'\'
    接收一个视频的url
    :param url: 
    :return: 
    \'\'\'
    try:
        response = requests.get(url)
        # response：请求到的资源
        if response.status_code == 200:
            m = hashlib.md5()
            m.update(str(time.time()).encode("utf-8"))
            m.update(url.encode("utf-8"))
            filepath = "%s\%s.mp4" % (movie_path, m.hexdigest())   # 视频名字是movie_path/时间字符串的哈希值的加密字符串
            with open(filepath, "wb") as f:
                f.write(response.content)                #文件是以wb模式打开，所以用content的方式写入
                print("%s 下载成功" % url)
    except Exception:
        pass


def main():
    \'\'\'
    url：格式化后的url字符串；
    index_page:第一次请求到的页面；
    detail_urls：页面中的url列表
    detail_page：上边列表中的url每个发送一次get请求
    movie_urls：解析后的视频地址
    :return: 文件写入硬盘
    \'\'\'
    base_url = \'http://www.xiaohuar.com/list-3-{page_num}.html\'     # 请求地址
    for i in range(5):                                              # 视频总共有五页
        url = base_url.format(page_num=i)
        index_page = get_page(url)
        detail_urls = parse_index(index_page) 
        for detail_url in detail_urls:
            detail_page = get_page(detail_url)
            movie_urls = parse_detail(detail_page)
            for movie_url in movie_urls:
                get_movie(movie_url)

if __name__ == \'__main__\':
    main()

#并发爬取
import requests #pip3 install requests
import re
import hashlib
import time
from concurrent.futures import ThreadPoolExecutor

pool=ThreadPoolExecutor(50)
movie_path=r\'C:\mp4\'

def get_page(url):
    try:
        response=requests.get(url)
        if response.status_code == 200:
            return response.text
    except Exception:
        pass

def parse_index(index_page):
    index_page=index_page.result()
    urls=re.findall(\'class="items".*?href="(.*?)"\',index_page,re.S)
    for detail_url in urls:
        if not detail_url.startswith(\'http\'):
            detail_url=\'http://www.xiaohuar.com\'+detail_url
        pool.submit(get_page,detail_url).add_done_callback(parse_detail)

def parse_detail(detail_page):
    detail_page=detail_page.result()
    l=re.findall(\'id="media".*?src="(.*?)"\',detail_page,re.S)
    if l:
        movie_url=l[0]
        if movie_url.endswith(\'mp4\'):
            pool.submit(get_movie,movie_url)

def get_movie(url):
    try:
        response=requests.get(url)
        if response.status_code == 200:
            m=hashlib.md5()
            m.update(str(time.time()).encode(\'utf-8\'))
            m.update(url.encode(\'utf-8\'))
            filepath=\'%s\%s.mp4\' %(movie_path,m.hexdigest())
            with open(filepath,\'wb\') as f:
                f.write(response.content)
                print(\'%s 下载成功\' %url)
    except Exception:
        pass

def main():
    base_url=\'http://www.xiaohuar.com/list-3-{page_num}.html\'
    for i in range(5):
        url=base_url.format(page_num=i)
        pool.submit(get_page,url).add_done_callback(parse_index)

if __name__ == \'__main__\':
    main()