爬取哔哩哔哩每日热榜

#哔哩哔哩每日热榜
#网页链接：https://www.bilibili.com/ranking?
import requests
from bs4 import BeautifulSoup
#发出request请求，获取html网页
url = \'https://www.bilibili.com/ranking?\'
kv = {\'user-agent\': \'Mozilla/5.0\'}#伪装爬虫
r = requests.get(url,timeout = 30,headers=kv)
r.text#获取源代码
html=r.text
#解析网页，提取内容
soup=BeautifulSoup(html,\'lxml\')#构造Soup的对象
res = soup.find_all(\'a\',class_=\'title\')
num = 0
text = \'\'
for i in res:
    num+=1
    text+=\'{}{}\n\'.format(num,i.string)#先把内容保存到变量里去
print(text)
#保存
with open(\'rank.text\',\'w\',encoding=\'utf8\')as fout:
    fout.write(text)

 

2020.3.26

import requests
from bs4 import BeautifulSoup
import csv
import datetime
url = \'https://www.bilibili.com/ranking?\'
#发起网络请求
response = requests.get(url)
html_text = response.text
soup = BeautifulSoup(html_text,\'html.parser\')
def main():
    #用来保存视频信息的对象
    class Video:
        def __init__(self,rank,title,point,visit,up,url):
            self.rank = rank
            self.title = title
            self.point = point
            self.visit = visit
            self.up = up
            self.url = url
        def to_csv(self):
            return [self.rank,self.title,self.point,self.visit,self.up,self.url]

        @staticmethod
        def csv_title():
            return [\'排名\',\'标题\',\'分数\',\'播放量\',\'UP\',\'URL\']
    #提取列表
    items = soup.find_all(\'li\',{\'class\':\'rank-item\'})
    videos = [] #保存提取出来的video
    for itm in items:
        title = itm.find(\'a\',{\'class\':\'title\'}).text#标题
        point = itm.find(\'div\',{\'class\':\'pts\'}).text#综合得分
        rank = itm.find(\'div\',{\'class\':\'num\'}).text#排名
        visit = itm.find(\'span\',{\'class\':\'data-box\'}).text#播放量
        up = itm.find_all(\'a\',)[2].text#up
        url = itm.find(\'a\',{\'class\':\'title\'}).get(\'href\')#获取链接
        v = Video(rank,title,point,visit,up,url)
        videos.append(v)

    #保存
    now_str = datetime.datetime.now().strftime(\'%Y%m%d\')
    file_name = f\'top100.csv_{now_str}.csv\'
    with open(file_name,\'w\',newline=\'\') as f:
        writer = csv.writer(f)
        writer.writerow(Video.csv_title())
        for v in videos:
            writer.writerow(v.to_csv())

 

1.打开网页

2.获取源代码

3.解析网页，提取需要的内容，先找第一名的

 

 

 

这里找到需要提取的标题a标签，分析特点，它的类是title，在代码中可以用find_all函数查找

 

发现成功将排行榜爬取下来，想到可以用for循环把结果一个个打印出来

 

 

因为内容都是按顺序排下来的，所以可以自己弄数字形成排名

 

 

然后把内容保存到一个变量里去并检查有没有正常保存

 

 

 

 

最后直接保存到文件里面去，创建一个rank.txt,以写入的方式打开，把它赋值到fout这个变量里，fout写入获取到的文本内容

 

 

获取数据截图