爬取某电影网站 新

主类

import re
import requests
import time
from scrapy import Selector
from Film_website.models import Film
# 
url = \'https://www.88ys.cc\'

def search(name, num = 1):
    r = requests.get(\'https://www.88ys.cc/vod-search-pg-{}-wd-{}.html\'.format(num, name))
    sel = Selector(text=r.text)
    results = sel.xpath(\'//div[@class="index-area clearfix"]/ul//li[@class="p1 m1"]\').extract()
    result_ = len(results)
    if result_:
        pages = sel.xpath(\'//div[@class="page mb clearfix"]/text()\').extract_first()
        page = int(re.search(\'(\d*)/(\d*)\',pages).group(1))
        pagecount = int(re.search(\'(\d*)/(\d*)\',pages).group(2))
        if page <= pagecount:
            for result in results:
                sel_result = Selector(text=result)
                href = url + (sel_result.xpath(\'//a/@href\').extract_first())
                title = sel_result.xpath(\'//p[@class="name"]/text()\').extract_first()
                type_ = sel_result.xpath(\'//span[@class="lzbz"]/p[3]/text()\').extract_first()
                time_ = sel_result.xpath(\'//span[@class="lzbz"]/p[4]/text()\').extract_first()
                try:
                    address =Film.filter(address=href)
                    print(\'此信息已录入: {} {} \'.format(address[0].id,address[0].film_name))
                    print(\'--------------------\')
                except IndexError:
                    Film.create(address=href, film_name=title, type_=type_, time_=time_)
                    print(href)
                    print(title)
                    print(type_)
                    print(time_)
                    print(\'--------------------\')
            print(\'-----------当前页数:{}/{}-----------\'.format(page, pagecount))
            # time.sleep(3)
            return search(name, num+1)
    return print(\'循环结束\')


def query(name):
    films = Film.select().where(Film.film_name == name)
    for film in films:
        print(film.film_name, \' \', film.address)
        # return film.film_name, film.address

def details():
    #数据库中获取电影详情链接
    #获取播放源(前两个)
    #获取集数
    #打开第一集获取所有集数播放地址和m3u8地址
    #存储数据库或者下载或者(未定)
    return


if __name__==\'__main__\':
    #2019/8/7 总页数2939
    #2019/8/8 总页数2940
    search(\'0\')
    # query(\'家有女友\')

数据库类

from peewee import *


#建立数据库连接
db = MySQLDatabase(\'spider\', host=\'127.0.0.1\', port=3306, user=\'root\', password=\'123456lmr\')
#

class BaseModel(Model):
    class Meta:
        database = db

class Film(BaseModel):
    # id = IntegerField(primary_key=True)
    address = CharField(default=\'空\')
    film_name = CharField(max_length=50, default=\'空\')
    type_ = CharField(max_length=50, default=\'空\')
    time_ = CharField(max_length=50, default=\'空\')


if __name__ == \'__main__\':
    # db.create_tables([Good, GoodEvaluate, GoodEvaluateSummary])
    # db.create_tables([Goods])
    db.create_tables([Film])

 

重点

网站抓取完后,我通过各种途径了解了一下这种网站的视频来源, 结果发现有个专门的采集网站, 来采集这些视频, 这种网站就是空壳, 只是存储采集网站的视频链接, 如果最开始就知道采集网站的话, 就不用这么麻烦了[捂脸哭]