python抓取页面内容实例

程序目的：

前段时间弄了个论坛(http://www.yyjun.net)，懒于手动找贴发帖，特写个抓取指定url中帖子内容的小程序(完善中)

已经实现功能：

由于我们需要抓取页面特定DOM下的链接，如程序中，我们抓取

http://news.sina.com.cn/society/ 页面中大标题的链接，因此可以用函数getlink获取，返回list型结果，代码如下：

url="http://news.sina.com.cn/society/"
sinaurls=getlink(url, \'h1\', {})，花括号为我们指定了附属条件，可以添加附加条件限制

返回结果[url1, url2]

接下来就是获取页面中的内容，如：获取新闻标题、内容(主要函数getcontent)，代码如下

getcontent(u, \'h1\', {\'id\':\'artibodyTitle\'}, subtag=None).encode(\'utf-8\')
getcontent(u, \'div\', {\'id\':\'artibody\'}, subtag=None).encode(\'utf-8\')

subtag=None代表前面的参数已经可以获取到结果了，否则的话指定子DOM节点tagname和tagattr属性

这样我们就获取到了所要的数据，不过现在程序还不太灵活，接下来要完成的事情：

1. 数据源的配置（url,tagname, tagattr等）

2. 如何定时抓取

学习资料：

Python简明教程， python手册

开发环境：

WindowXP + Python2.6 + BeautifulSoup + Mysql5

#coding=utf-8
#spider
#2011-9-8 23:06

from datetime import datetime
from BeautifulSoup import BeautifulSoup
from urllib2 import urlopen, Request,re
import MySQLdb
import sys

class mysql:
    def __init__(self, host=\'localhost\', user=\'root\', pwd=\'\', db=\'\'):
        try:
            self.db = MySQLdb.connect(host=host,user=user,passwd=pwd,db=db,charset="utf8", use_unicode=True)
            self.cursor=self.db.cursor()
            self.cursor.execute("set names utf8")
            self.db.commit()
        except Exception, e:
            print e

    def execute(self, sql):
        try:
            self.cursor.execute("set names utf8")
            self.cursor.execute(sql)
        except Exception, e:
            print e
    def fetchone(self):
        try:
            row = self.cursor.fetchone()
            return row[0]
        except Exception, e:
            print e
    def commit(self):
        self.db.commit()
    def rollback(self):
        self.db.rollback()

def getsoup(url):
    req=Request(url)
    try:
        rep=urlopen(req)
    except IOError:
        print \'不能链接到指定地址，请确认%s可访问\' % url
        return None
    soup=BeautifulSoup(rep.read())
    return soup

def getlink(url, tag, attr={}, subtagattr={}):
    \'\'\'
    获取url页面中DOM元素下的所有链接
    url:地址
    tag:一级DOM节点的tagname
    attr:一级DOM节点的附加属性
    subtagattr:子孙DOM节点的附加属性
    \'\'\'
    soup=getsoup(url)

    mainnode=soup.findAll(tag, attrs=attr)
    linknodes=[]
    for mn in mainnode:
        linknodes = linknodes + [n for n in mn.findAll(\'a\', attrs=subtagattr) if n not in linknodes]

    urls=[]
    for l in linknodes:
        urls.append(l[\'href\'])
    return urls;

def getcontent(url, tag, attr={}, subtag=\'a\', subattr={}):
    soup=getsoup(url)

    node=soup.find(tag, attr)
    if subtag != None:
        node=node.find(subtag, subattr)
    contents=node.contents
    contents=[unicode(n) for n in contents if True]
    return filtercontent(u\'\'.join(contents))

def getcontentlist(url, tag, attr={}, subtag=\'a\', subattr={}):
    soup=getsoup(url)

    node=soup.findAll(tag, attr)
    if subtag != None:
        node=node[0].findAll(subtag, subattr)
    l=[]
    for n in node:
        l.append(u\'\'.join([unicode(c) for c in n.contents if True]))
    return l


def filtercontent(content):
    r0=re.compile(\'\n\')
    r1=re.compile(\'\')
    r2=re.compile(\'<script.*?>.*?</script>\')
    r3=re.compile(\'<style.*?>.*?</style>\')
    content=r0.sub(\'\', content)
    content=r1.sub(\'\', content)
    content=r2.sub(\'\', content)
    content=r3.sub(\'\', content)
    return content

if __name__ == \'__main__\':
    INSERT_FORUM_THREAD_SQL=\'\'\'
                INSERT INTO forum_thread(fid, author, authorid, subject, dateline, lastpost, lastposter)
                values(38, \'小俊\', 11, \'%s\', UNIX_TIMESTAMP(now()), UNIX_TIMESTAMP(now()), \'小俊\')
                \'\'\'
    LAST_INSERT_ID_SQL = \'SELECT LAST_INSERT_ID()\'
    INSERT_FORUM_POST_SQL=\'\'\'
                INSERT INTO forum_post
                (fid, tid, first, author, authorid, subject, dateline, message,useip, usesig, htmlon, bbcodeoff)
                VALUES
                (38, %s, 0, \'小俊\', 11, \'%s\', UNIX_TIMESTAMP(now()), \'%s\', \'125.39.155.30\', 1, 1, 1)
                \'\'\'
    UPDATE_FORUM_FORUM_SQL=\'\'\'
                UPDATE forum_forum
                SET
                threads=threads+%s,
                posts=posts+%s,
                lastpost=concat(\'%s    %s    \' ,CAST(UNIX_TIMESTAMP(now()) AS CHAR(50)),\'    小俊\'),
                todayposts=%s
                WHERE fid = 38
                \'\'\'

    db=mysql(pwd=\'abc\', db=\'test\')
    url="http://news.sina.com.cn/society/"
    sinaurls=getlink(url, \'h1\', {})
    tid=\'\'
    title=\'\'
    timeline=\'\'
    author=\'\'
    i=0
    try:
        for u in sinaurls:
            title=MySQLdb.escape_string(getcontent(u, \'h1\', {\'id\':\'artibodyTitle\'}, subtag=None).encode(\'utf-8\'))
            body=MySQLdb.escape_string(getcontent(u, \'div\', {\'id\':\'artibody\'}, subtag=None).encode(\'utf-8\'))
            db.execute(INSERT_FORUM_THREAD_SQL % title)
            db.execute(LAST_INSERT_ID_SQL)
            insertid=db.fetchone()
            db.execute(INSERT_FORUM_POST_SQL % (insertid, title, body))
            tid=insertid
            title=title
            i=i+1
        db.execute(UPDATE_FORUM_FORUM_SQL % (i, i, tid, title, i))
        db.commit()
    except:
        db.rollback()