python爬虫下载小视频和小说(基础)

下载视频：

 1 from bs4 import BeautifulSoup
 2 import requests
 3 import re
 4 import urllib
 5 
 6 
 7 def callbackfunc(blocknum, blocksize, totalsize):
 8     \'\'\'回调函数
 9     @blocknum: 已经下载的数据块
10     @blocksize: 数据块的大小
11     @totalsize: 远程文件的大小
12     \'\'\'
13     percent = 100.0 * blocknum * blocksize / totalsize
14     if percent > 100:
15         percent = 100
16     print ("%.2f%%"% percent)
17 
18 
19 
20 ur = \'http://www.budejie.com/video/\'
21 
22 def get_htmls(url):
23     headers = {\'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36\'}
24     html = requests.get(url,headers=headers)
25     #print(type(html.text))
26     soup =BeautifulSoup(html.text,\'html.parser\')
27     result1 = soup.find(attrs={\'class\':\'j-video-c\',\'data-title\':True})
28     result2 = soup.find(attrs={\'class\': \'j-video\', \'data-mp4\': True})
29     nam = result1.get(\'data-title\')
30     url = result2.get(\'data-mp4\')
31     local = \'e:\\\'+str(nam)+\'.mp4\'
32     urllib.request.urlretrieve(url, local, callbackfunc)
33 
34 if __name__ == \'__main__\':
35     get_htmls(ur)

 

下载小说：

 1 from bs4 import BeautifulSoup
 2 import requests
 3 import re
 4 from openpyxl import load_workbook
 5 from openpyxl.utils import get_column_letter
 6 
 7 #这一部分是存链接的
 8 \'\'\'
 9 # 设置文件 mingc
10 addr = "1.xlsx"
11 # 打开文件
12 wb = load_workbook(addr)
13 # 创建一张新表
14 ws = wb.create_sheet()
15 # 第一行输入
16 #ws.append([\'TIME\', \'TITLE\', \'A-Z\'])
17 ws[\'A1\'] = \'章节\'
18 ws[\'B1\'] = \'链接
19 \'\'\'
20 links = []
21 ur = \'https://www.qb5200.tw/xiaoshuo/2/2155/\'
22 def get_one_page(url,ok):
23     headers = {
24 
25    \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763\'}
26 
27     html = requests.get(url,headers=headers)
28     html.encoding = \'gbk\'
29     #print(html.content)
30     if ok == True:
31         get_parsing(html)
32     else :
33         return html
34 
35 def get_parsing(html):
36     soup = BeautifulSoup(html.content,\'html.parser\')
37     dd = soup.findAll([\'dt\',[\'dd\']])
38     result = False
39 
40     #counts =2
41     co = 0
42     for one in dd:
43         #print(type(one))
44         if one.string==\'《龙王传说》正文卷\':
45             #ws.title = one.string
46             result = True
47         if result == True and one.name == \'dd\':
48             link = one.a.get(\'href\')
49             links.append(link)   #注意这里是传入元组、列表、字典
50 \'\'\'
51             st = one.a.string
52             data = [++co,ur+link]
53             ws.append(data)
54 \'\'\'
55 
56 def get_htmls():
57     i=1000
58     results = links[1000:]
59     for link in results:
60         i+=1
61         url = ur+link
62         path=\'龙3.txt\'
63         html = get_one_page(url,False)
64         soup = BeautifulSoup(html.content, \'html.parser\')
65         name = soup.find(attrs={\'class\':\'content\',\'id\':False})
66         names = name.h1.string
67         div = soup.find(\'div\',attrs={\'class\':\'showtxt\'})
68         with open(path,\'a\',encoding=\'utf8\') as f:
69             f.write(names + \'\n\')
70             for string in div.stripped_strings:
71                 f.write(string+\'\n\')
72 
73         if i%10 == 0:
74             print(i)
75             if i==1300:
76                 break
77 
78 
79 
80 if __name__ == \'__main__\':
81     get_one_page(\'https://www.qb5200.tw/xiaoshuo/2/2155/\',True)
82     #wb.save(addr)
83     get_htmls()