scrapy框架之cookie和代理操作

一，scrapy发送post请求

scrapy框架中默认发送的是get请求，源码：

scrapy.Spider父类中的start_request方法：

    def start_requests(self):
        cls = self.__class__
        if method_is_overridden(cls, Spider, \'make_requests_from_url\'):
            warnings.warn(
                "Spider.make_requests_from_url method is deprecated; it "
                "won\'t be called in future Scrapy releases. Please "
                "override Spider.start_requests method instead (see %s.%s)." % (
                    cls.__module__, cls.__name__
                ),
            )
            for url in self.start_urls:
                yield self.make_requests_from_url(url)
        else:
            for url in self.start_urls:
                yield Request(url, dont_filter=True)

那么，想要发送post请求，我们就需要在我们的爬虫文件中重写父类的start_request方法。

详见代码：

ex:利用爬虫发送post请求到百度翻译

import scrapy


class PosttestSpider(scrapy.Spider):
    name = \'postTest\'
    # allowed_domains = [\'www.qiubai.com\']
    start_urls = [\'http://www.qiubai.com/\']

    def start_requests(self):
        url = \'https://fanyi.baidu.com/sug\'
        data = {
            "kw": "hello"
        }
        yield scrapy.FormRequest(url=url, formdata=data, callback=self.parse)

    def parse(self, response):
        print(response.text)

二，cookie

scrapy框架中发送的request请求，默认回保留cookie。

ex:利用爬虫登陆豆瓣电影，获取个人主页

# -*- coding: utf-8 -*-
import scrapy
from cookieDemo.utils.YMD import YDMHttp


class DoubanSpider(scrapy.Spider):
    name = \'douban\'
    # allowed_domains = [\'www.douban.com\']
    start_urls = [\'https://www.douban.com/\']

    def parse(self, response):
        img_code_url = response.xpath("//*[@id=\'captcha_image\']/@src").extract_first()
        captcha_id = response.xpath("//*[@id=\'lzform\']/fieldset/div[3]/div/input[2]/@value").extract_first()
        print(img_code_url)
        print(captcha_id)
        yield scrapy.Request(url=img_code_url, callback=self.parse_img, meta={"captcha_id": captcha_id})  # 利用meta传参

    def parse_img(self, response):
        with open("code.png", "wb") as f:
            f.write(response.body)

        # 发送登陆请求
        login_url = "https://www.douban.com/accounts/login"
        img_code = self.get_code("code.png")
        captcha_id = response.meta.get("captcha_id")  # 利用request传参
        data = {
            \'redir\': \'https://movie.douban.com/\',
            "source": "movie",
            "form_email": "glh0220@qq.com",
            "form_password": "goulonghui371379.",
            "captcha-solution": img_code,
            "captcha-id": captcha_id,
            \'login\': \'登录\',
        }
        yield scrapy.FormRequest(url=login_url, formdata=data, callback=self.parse_login)

    def parse_login(self, response):
        people_url = "https://www.douban.com/people/186597252/"
        yield scrapy.Request(url=people_url, callback=self.get_people_page)

    def get_people_page(self, response):
        with open("people.html", "w", encoding="utf-8") as f:
            f.write(response.text)
            print("over...............................")

    def get_code(self, img_path):
        # 用户名
        username = \'EksYiQiang\'
        # 密码
        password = \'xyq19990113\'

        # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
        appid = 6041

        # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
        appkey = \'c9f0265f96d9e97118aeb8eff629da64\'

        # 图片文件
        filename = img_path

        # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
        codetype = 3000

        # 超时时间，秒
        timeout = 60

        # 检查
        if (username == \'username\'):
            print(\'请设置好相关参数再测试\')
            return
        else:
            # 初始化
            yundama = YDMHttp(username, password, appid, appkey)

            # 登陆云打码
            uid = yundama.login()
            print(\'uid: %s\' % uid)

            # 查询余额
            balance = yundama.balance()
            print(\'balance: %s\' % balance)

            # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果
            cid, result = yundama.decode(filename, codetype, timeout)
            print(\'cid: %s, result: %s\' % (cid, result))
        return result

三，代理操作

ex:利用百度可以搜索ip可以显示本机ip,测试代理操作

import scrapy


class IpdemoSpider(scrapy.Spider):
    name = \'IPdemo\'
    allowed_domains = [\'www.baidu.com\']
    start_urls = [\'https://www.baidu.com/s?wd=ip\']

    def parse(self, response):
        with open("ip.html", "w", encoding="utf-8") as f:
            f.write(response.text)
            print("over")

DownloadMiddleware:

class MyProxyMiddleware(object):

    def process_request(self, request, spider):
        # 这里的request就是就是中间件拦截的请求对象
        # 该方法可以拦截请求对象
        # 将该请求对象的UA进行伪装
        # 对该请求的uri进行篡改
        request.meta["proxy"] = "https://151.106.15.12:1080"

settings.py:

DOWNLOADER_MIDDLEWARES = {
   # \'ipDemo.middlewares.IpdemoDownloaderMiddleware\': 543,
   \'ipDemo.middlewares.MyProxyMiddleware\': 543,
}