python爬虫-搜索小说并下载

  1 #coding:utf-8
  2 import requests,os,re
  3 from bs4 import BeautifulSoup
  4 from selenium import webdriver
  5 from selenium.webdriver.chrome.options import Options
  6 from selenium.webdriver.common.keys import Keys
  7 
  8 class downloader():
  9 
 10     def __init__(self):
 11         self.urls = []  # 保存章节链接
 12         self.name = []  # 保存章节名
 13         self.url = \'https://so.biqusoso.com/s.php?ie=utf-8&siteid=biqugex.com&q=\'
 14 
 15     """输入小说名，搜索"""
 16     def Get_url(self):
 17         #创建chrome参数对象，设置chrome浏览器无界面模式
 18         chrome_options = Options()
 19         chrome_options.add_argument(\'--headless\')
 20         # 创建chrome无界面对象
 21         browser = webdriver.Chrome(options=chrome_options)
 22         browser.get(self.url)
 23         c = input(\'请输入小说全名：\')
 24         browser.find_element_by_xpath(\'//*[@id="wrapper"]/div[1]/div[2]/form/input[3]\').send_keys(c)
 25         browser.find_element_by_xpath(\'//*[@id="wrapper"]/div[1]/div[2]/form/input[4]\').click()
 26         new_url = browser.current_url
 27         # 关闭浏览器
 28         browser.close()
 29         # 关闭chromedriver进程
 30         browser.quit()
 31         print("已关闭浏览器")
 32         # print(new_url)
 33         response = requests.get(new_url)
 34         response.encoding = \'utf-8\'
 35         soup = BeautifulSoup(response.text, \'lxml\')
 36         # print(soup)
 37         name1 = soup.find_all(\'span\', class_=\'s2\')
 38         soup = BeautifulSoup(str(name1), \'lxml\')
 39         new_name = soup.find(\'a\')
 40         new_name1 = new_name.string
 41         # print(new_name1)
 42         self.href = new_name.attrs[\'href\']
 43         print(self.href)
 44         return self.href
 45     def Response(self):
 46         response = requests.get(self.href)
 47         response.encoding = \'gbk\'  # 解决乱码
 48         self.soup = BeautifulSoup(response.text, \'lxml\')  # 解析网页
 49         div = self.soup.find_all(\'div\', class_=\'listmain\')  # 在解析结果中查找class_=\'listmain\'
 50         soup1 = BeautifulSoup(str(div), \'lxml\')  # 删除字符串头和尾的空格
 51         h = soup1.find_all(\'a\')  # 在class_=\'listmain下面找到a标签
 52         for i in h:
 53             self.name.append(i.string)  # 将a标签中的非属性字符，即章节名添加到name
 54             self.urls.append(\'https://www.biqugex.com%s\' % i.get(\'href\'))  # 将a标签中的链接，添加到urls
 55 
 56     def file(self):
 57         """查找小说名字，并创建同名文件夹"""
 58         div1 = self.soup.select(\'body > div.book > div.info > h2\')
 59         a = BeautifulSoup(str(div1), \'lxml\')
 60         b = a.find(\'h2\')
 61         b = b.string
 62         c = \'C:\\Users\\Administrator\\Desktop\\%s\' % b
 63         if not os.path.exists(c):
 64             os.mkdir(c)
 65 
 66         # 循环解析urls，得到小说正文
 67         i = 0
 68         while i < len(self.urls):
 69             response1 = requests.get(url=self.urls[i])
 70             response1.encoding = \'gbk\'
 71             soup2 = BeautifulSoup(response1.text, \'lxml\')
 72             d = soup2.find_all(\'div\', id=\'content\')
 73             id1 = BeautifulSoup(str(d), \'lxml\')
 74             # 创建文件名
 75             src = self.name[i] + \'.txt\'
 76             filename = c + \'/\' + src
 77             print(filename)
 78 
 79             # 将解析到的小说正文写到文件中
 80             for result in id1:
 81                 res = result.text
 82                 id2 = soup2.select(\'#content\')
 83                 with open(filename, \'w+\', encoding=\'utf-8\') as f:
 84                     f.write(res)
 85                 i += 1
 86 #如果输入的网址不是正确的网址，则提示请输入正确的笔趣阁网址
 87     def Main(self):
 88         try:
 89             d = downloader()
 90             d.Get_url()
 91         except:
 92             print(\'没有找到\')
 93         else:
 94             d.Response()
 95             d.file()
 96 
 97 
 98 
 99 if __name__ == \'__main__\':
100     # url=input(\'请输入网址：\')
101     # url=\'https://www.biqugex.com/book_104027/\'
102     a = downloader()
103     a.Main()