xpath爬取喜马拉雅糗事播报音频地址

---

获取url地址，设置headers,使用xpath对网页进行解析：

 

 

 

 

 

 

 

---

 

接受介绍页的url,通过xpath解析出来每一个音频的id值，标题

 

 

 

 

下载内容：

 

 

 

 

---

 

代码如下

 

import requests
from lxml import etree
import time

headers = {
    \'authority\': \'mermaid   .ximalaya.com\',
    \'cache-control\': \'no-cache\',
    \'upgrade-insecure-requests\': \'1\',
    \'user-agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36\',
    \'accept\': \'*/*\',
    \'sec-fetch-site\': \'same-site\',
    \'sec-fetch-mode\': \'cors\',
    \'sec-fetch-user\': \'?1\',
    \'sec-fetch-dest\': \'empty\',
    \'referer\': \'https://www.ximalaya.com/yule/3493173/\',
    \'accept-language\': \'zh-CN,zh;q=0.9\',
    \'cookie\': \'_xmLog=h5^&2e54f20f-24dc-48df-a0c0-5e4d5bc28df5^&2.1.2; x_xmly_traffic=utm_source^%^253A^%^2526utm_medium^%^253A^%^2526utm_campaign^%^253A^%^2526utm_content^%^253A^%^2526utm_term^%^253A^%^2526utm_from^%^253A; Hm_lvt_4a7d8ec50cfd6af753c4f8aee3425070=1608252623,1608253346,1608253440,1608253957; Hm_lpvt_4a7d8ec50cfd6af753c4f8aee3425070=1608253968\',
    \'Referer\': \'https://www.ximalaya.com/yule/3493173/\',
    \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36\',
    \'Origin\': \'https://www.ximalaya.com\',
    \'if-modified-since\': \'Wed, 28 Nov 2018 07:43:22 GMT\',
    \'Connection\': \'keep-alive\',
    \'Accept\': \'*/*\',
    \'Sec-Fetch-Site\': \'cross-site\',
    \'Sec-Fetch-Mode\': \'no-cors\',
    \'Sec-Fetch-Dest\': \'script\',
    \'Accept-Language\': \'zh-CN,zh;q=0.9\',
    \'Content-Type\': \'application/x-www-form-urlencoded;charset=UTF-8\',
    \'xm-sign\': \'24393343084be486d4ce4228bc83f4a8(37)0(18)1608256264690\',
    \'content-type\': \'application/octet-stream\',
    \'origin\': \'https://www.ximalaya.com\',
    \'If-None-Match\': \'f45020c793859650e0da4f3720059922\',
    \'pragma\': \'no-cache\',
}


def get_html(url):
    time.sleep(0.5)
    response = requests.get(url, headers=headers)
    response.encoding = response.apparent_encoding
    return response

def html_parse(text):
    tree = etree.HTML(text)
    return tree

def save(url, name):
    print(url)
    response = requests.get(url)
    with open("./糗事播报/%s.m4a" % name, "wb") as f:
        f.write(response.content)

def get_m4a(id, title):
    url = \'https://www.ximalaya.com/revision/play/v1/audio?id=%s&ptype=1\' % id
    response = get_html(url).json()
    yinpin_url = response["data"]["src"]
    print("开始下载----%s" % title)
    save(yinpin_url, title)
    print("下载完成----%s" % title)



def main(url):
    """
        接收介绍页的url，通过xpath解析出来每一个音频的id值、标题。
    :param url: 介绍页的url
    :return: 
    """
    response = get_html(url)
    tree = html_parse(response.text)
    res_urls = tree.xpath(\'//*[@id="anchor_sound_list"]/div[2]/ul/li/div[2]/a/@href\')
    yinpin_titles = tree.xpath(\'//*[@id="anchor_sound_list"]/div[2]/ul/li/div[2]/a/@title\')
    for res, yinpin_title in zip(res_urls, yinpin_titles):
        get_m4a(res.split("/")[-1], yinpin_title)


if __name__ == \'__main__\':
    for i in range(10):
        if i == 0:
            url = "https://www.ximalaya.com/yule/3493173/"
        else:
            url = "https://www.ximalaya.com/yule/3493173/p%s/" % (i+1)
        print("-----开始下载第%s页内容------" % (i+1))
        main(url)