python爬虫：爬取医药数据库drugbank

这个是帮朋友做的，难点就是他们有一个反爬虫机制，用request一直不行，后面我就用selenium直接把网页copy下来，然后再来解析本地的html文件，就木有问题啦。

现在看来，写得有点傻，多包涵。

 

# -*- coding:utf-8 -*-

import os
import time
import datetime
import codecs
from lxml import etree
from selenium import webdriver
import csv
#控制编码，全英文网页，用不着
# import sys
# reload(sys)
# sys.setdefaultencoding(\'utf-8\')

# # date格式转为string格式
today = datetime.date.today()
today_string = today.strftime(\'%Y-%m-%d\')

#通过浏览器得到网页页面--反反爬虫
def html_getter(site,file_name):
    driver = webdriver.Firefox()
# chromedriver = r\'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe\'
    # os.environ[\'webdriver.chrome.driver\'] = chromedriver
    # driver = webdriver.Chrome(chromedriver)
    driver.get(site)
    driver.maximize_window() # 将浏览器最大化显示
    time.sleep(5) # 控制间隔时间，等待浏览器反映
    # 保存页面
    source_code = driver.find_element_by_xpath("//*").get_attribute("outerHTML")
    f = codecs.open(file_name, \'w+\', \'utf8\')
    f.write(source_code)
    f.close()

#打开保存在本地的html文件
def file_html(file_name):
    f = open(file_name,\'r\')
    html = f.read()
    f.close()
return html

#写入csv，也可以有其他写入方式，这个地方就csv啦
def csv_writer(ll):
    headers = [\'drug\',\'inter\',\'snp_rs_id\',\'Allele_name\',\'Defining_change\',\'Adverse_Reaction\',\'ref\',\'href\',\'original_title\']
with open(\'drugbank.csv\',\'a\') as f:
        f_csv = csv.writer(f)
        f_csv.writerow(headers)
        f_csv.writerows(ll)

#用xpath解析网页，得到表格数据，我就是这么爱xpath，不喜欢正则表达式
def data_get(html):
    selector = etree.HTML(html)
    tbody=selector.xpath(\'/html/body/main/table/tbody/tr\')
for each in tbody:
# #1.\'drug\'
        drug_name=each.xpath(\'td[1]/strong/text()\')[0]
        drug_sn=each.xpath(\'td[1]/a/text()\')[0]
        drug=drug_name+\'   \'+drug_sn
# #print(drug)
        # #2.\'Interacting Gene/Enzyme\'
        int=each.xpath(\'td[2]\')[0]
        inter=int.xpath(\'string(.)\')
# print(inter)
        # #3.\'SNP RS ID\'
        snp=each.xpath(\'td[3]/a/text()\')
if snp:
            snp_rs_id=snp[0]
else:
            snp_rs_id=\'Not Available   \'
        #print snp_rs_id
        #4.Allele name
        Allele=each.xpath(\'td[4]/text()\')
if Allele:
            Allele_name=Allele[0]
else:
            Allele_name=\'Not Available \'
        # #print Allele_name
        # #5.\'Defining change\'
        Defining=each.xpath(\'td[5]/text()\')
if Defining:
            Defining_change=Defining[0]
else:
            Defining_change=\'Not Available \'
        # print Defining_change
        # 6.\'Adverse Reaction\'
        Adverse=each.xpath(\'td[6]/text()\')
if Adverse:
            Adverse_Reaction=Adverse[0]
else:
            Adverse_Reaction=\'Not Available    \'
        # print Adverse_Reaction
        #7.\'Reference(s)\'
        ref=each.xpath(\'td[7]/span/a/text()\')[0]
        href=each.xpath(\'td[7]/span/a/@href\')[0]
        original_title=each.xpath(\'td[7]/span/a/@data-original-title\')[0]
# print ref
        # print(href)
        # print(original_title)

        tt=(drug,inter,snp_rs_id,Allele_name,Defining_change,Adverse_Reaction,ref,href,original_title)
        ll.append(tt)

#print ll



if __name__ == \'__main__\':
    ll=[]
for i in range(1,5):
        page_num=i
        site=\'http://www.drugbank.ca/genobrowse/snp-adr?page=\'+str(page_num)
#get the html through webdriver
        file_name=unicode(today_string)+u\'drugbank_\'+unicode(str(page_num))+u\'.html\'

        html_getter(site,file_name)
        html=file_html(file_name)
        data_get(html)
    csv_writer(ll)