Python之爬取网页的一个例子

import time,random
import urllib2,urllib,socket,re
from bs4 import BeautifulSoup
import cx_Oracle



conn = cx_Oracle.connect(\'xxx/xxx\')
try:
    cursor = conn.cursor()
    cursor.execute(\'create table tb_user(id varchar2(50), name varchar2(50),password varchar(50))\')
except:
    print "wwwwwwww"
x = 0
my_dh = 0

def crawl(url):

    headers = { \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\', }
    req = urllib2.Request(url, headers=headers)
    page = urllib2.urlopen(req, timeout=60)
    contents = page.read()

    soup = BeautifulSoup(contents, \'html.parser\')
    my_title = soup.select(".des h2 a")
    file = open(\'E:\Python\\text.txt\', \'a\')

    # for i,z in zip(my_title,my_title2):
    #     b = i.text.strip()
    #     d = z.text.strip()
    #     # w = c.text.strip()
    #     n =  b+\'\'+d
    #     print n


    for phone in my_title:
        time.sleep(random.random()*5)
        url2 = phone[\'href\']
        html = urllib2.urlopen(url2).read()
        soup2 = BeautifulSoup(html, \'html.parser\')
        my_dh = soup2.select(\'.phone-num\')
        if len(my_dh)>0:
            my_dh1=my_dh[0].text
        else:
            my_dh1= \'null\'
            #continue

        my_man = soup2.select(\'.c_000\')
        if len(my_man)>0:
            my_man1 = soup2.select(\'.c_000\')[0].text
            my_bt = soup2.select(\'.c_333.f20\')[0].text
            my_money = soup2.select(\'.c_ff552e\')[0].text
            massage = url2 +\'  \'+ my_man1+\'  \'+my_dh1+\'  \'+my_bt + my_money

            print massage
            param = {\'id\': url2, \'n\': my_man1, \'p\': my_dh1}
            cursor.execute(\'insert into tb_user values(:id,:n,:p)\', param)
            conn.commit()
            print param
            file.write(massage.encode(\'utf-8\') + \'\n\')



        else:

            continue


for page in range(1, 100):
    page += 1
    url = \'http://cc.58.com/chuzu/pn{}\'.format(page)
    crawl(url)

能够将部分网页数据提取出来形成TXT文档。导入数据库时是每提取一条信息便导入oracle数据库。而导入文档时，则是提取一定数量的数据才会进行一次导入。

其中还有一些小毛病需要改善。