【python爬虫】2.爬小米应用商店

主要就是beautifulsoup的一些用法

但现在还是局限于：1.只能爬网页端 2.只能爬网页端存在超链接的地方

所以爬到的数据不多，要爬手机端还得接着学手机各种协议各种包的知识，还有很长的路

 

import time
import requests
import re
from bs4 import BeautifulSoup

def parse(soup):
    try:
        title=soup.body.find(class_=\'main\').find(class_=\'container cf\').find(class_=\'app-intro cf\').div.div.h3.text
        # print(title)
        fo.write(title+\'\n\')
    except:
        return
    details=soup.body.find(class_=\'main\').find(class_=\'container cf\')
    for i in details.find_all(style="width:100%; display: inline-block"):
        item=i.find(class_=\'float-left\').div.text
        data=i.find(class_=\'float-left\').find(style=\'float:right;\').text
        ans=item+\':\'+data
        b=ans.split()
        ans="".join(b)
        if ans:
            # print(ans)
            fo.write(ans+\'\n\')
    fo.write(\'\n\')

def get_url():
    for page in range(3):
        session = requests.Session()
        web1 = session.get(
            url=\'https://app.mi.com/catTopList/0?page=\'+str(page),
            headers={
                \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\',
            }
        )
        soup1 = BeautifulSoup(web1.text, \'lxml\')
        applist=soup1.body.find(class_=\'main\').find(class_=\'container cf\').find(class_=\'main-con\').find(class_=\'applist-wrap\').find(class_=\'applist\')
        for i in applist.find_all(\'li\'):
            # print(i.a.get(\'href\'))
            web2 = session.get(
                url=\'https://app.mi.com\'+i.a.get(\'href\'),
                headers={
                    \'User-Agent\': \'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\',
                }
            )
            soup2 = BeautifulSoup(web2.text, \'lxml\')
            parse(soup2)

if __name__ == \'__main__\':
    fo = open("mi.txt", "w")
    get_url()
    # parse(soup)

输出mi.txt：