简单python爬虫实例

先放上url，https://music.douban.com/chart

这是豆瓣的一个音乐排行榜，这里爬取了左边部分的歌曲排行榜部分，爬虫很简单，就用到了beautifulsoup和requests这两个库，爬取后分吧把内容存储到txt，csv和数据库

0x01：存储到txt

import requests
from bs4 import BeautifulSoup
url = \'https://music.douban.com/chart\'
html = requests.get(url=url).text
soup = BeautifulSoup(html, \'lxml\')
ul = soup.find(attrs={\'class\': \'col5\'})     # 找到存放排行榜音乐的ul标签
lis = ul.find_all(name=\'li\')                # 获取每一首音乐对应的标签
for li in lis:
    paiming = li.find(name=\'span\').string
    name = li.find(name=\'a\', attrs={\'href\': \'javascript:;\'}).string
    a = li.find(name=\'a\', attrs={\'class\': \'face\'})      # 获取存存放连接的a标签,再在a标签里获取连接
    if a != None:
        img = a.find(name=\'img\')    # 获取连接的img标签
        lianjie = img.attrs[\'src\']      # 获取到连接，下面进行一个判断，前十首歌曲有链接，后十首没有
    else:
        lianjie = \'没有链接\'
    print(paiming, name, lianjie)

    with open(\'dou_bai_music.txt\', \'a\', encoding=\'utf-8\') as f:
        f.write(paiming+\'\t\'+name+\'\t\'+lianjie+\'\n\')
        f.close()

0x02：存储到csv

import requests
from bs4 import BeautifulSoup
import csv
url = \'https://music.douban.com/chart\'
html = requests.get(url=url).text
soup = BeautifulSoup(html, \'lxml\')
ul = soup.find(attrs={\'class\': \'col5\'})     # 找到存放排行榜音乐的ul标签
lis = ul.find_all(name=\'li\')                # 获取每一首音乐对应的标签
with open(\'doubai.csv\', \'a\', newline=\'\', encoding=\'utf-8\') as f:　　　　# 打开文件，写入表头
    write = csv.writer(f)
    write.writerow([\'排名\', \'歌名\', \'链接\'])
for li in lis:
    paiming = li.find(name=\'span\').string
    name = li.find(name=\'a\', attrs={\'href\': \'javascript:;\'}).string
    a = li.find(name=\'a\', attrs={\'class\': \'face\'})      # 获取存存放连接的a标签,再在a标签里获取连接
    if a != None:
        img = a.find(name=\'img\')    # 获取连接的img标签
        lianjie = img.attrs[\'src\']      # 获取到连接
    else:
        lianjie = \'没有链接\'
    print(paiming, name, lianjie)
    with open(\'doubai.csv\', \'a\', newline=\'\', encoding=\'utf-8\') as f:　　# 打开文件，存储爬取内容
        write = csv.writer(f)
        write.writerow([paiming, name, lianjie])

0x03：存储到数据库

import requests
from bs4 import BeautifulSoup
import pymysql
url = \'https://music.douban.com/chart\'
html = requests.get(url=url).text
soup = BeautifulSoup(html, \'lxml\')
ul = soup.find(attrs={\'class\': \'col5\'})     # 找到存放排行榜音乐的ul标签
lis = ul.find_all(name=\'li\')                # 获取每一首音乐对应的标签
db = pymysql.connect(host=\'localhost\', user=\'root\', password=\'root\', port=3306, db=\'spiders\')　　# 连接数据库
cursor = db.cursor()
print(\'数据库连接成功\')
sql = \'CREATE TABLE IF NOT EXISTS music (paiming INT NOT NULL, name VARCHAR(255) NOT NULL, lianjie VARCHAR(255) NOT NULL, PRIMARY KEY (paiming))\'　　# 创建数据表，指定字段
cursor.execute(sql)
print(\'数据表创建完成！\')
for li in lis:
    paiming = li.find(name=\'span\').string
    name = li.find(name=\'a\', attrs={\'href\': \'javascript:;\'}).string
    a = li.find(name=\'a\', attrs={\'class\': \'face\'})      # 获取存存放连接的a标签,再在a标签里获取连接
    if a != None:
        img = a.find(name=\'img\')    # 获取连接的img标签
        lianjie = img.attrs[\'src\']      # 获取到连接
    else:
        lianjie = \'没有链接\'
    print(paiming, name, lianjie)
    sql = \'INSERT INTO music(paiming, name, lianjie) values(%s, %s, %s)\'　　　　# 插入数据，判断是否插入成功
    try:
        cursor.execute(sql, (paiming, name, lianjie))
        db.commit()
        print(\'数据插入完成！！\')
    except:
        print(\'插入失败\')
        db.rollback()
db.close()　　　　　　# 关闭数据库

 

*******************************不积跬步，无以至里。******************************