python 爬虫的初步实践

简介

我的sisiter，想要我爬一些试题给她。有80套，她不想手工点，所以，我来了，比较简单的网站。所以没有费很大的力气。期间参考了一系列的网站，都没有记录下来。

code

#!/usr/bin/env python
#coding=utf-8

import pdfkit
import time
import requests
import sys
import urllib2
import re

def get_hiddenvalue(url):
	request=urllib2.Request(url)
	reponse=urllib2.urlopen(request)
	resu=reponse.read()
	VIEWSTATE=re.findall(r\'<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="(.*?)" />\',resu, re.I)
	EVENTVALIDATION =re.findall(r\'input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="(.*?)" />\', resu,re.I)
	return VIEWSTATE[0],EVENTVALIDATION[0]

def get_hiddenvalue_string(myStr):
	VIEWSTATE=re.findall(r\'<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="(.*?)" />\',myStr, re.I)
	EVENTVALIDATION =re.findall(r\'input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION" value="(.*?)" />\', myStr,re.I)
	return VIEWSTATE[0],EVENTVALIDATION[0]

reload(sys)
sys.setdefaultencoding( "utf-8" )
data = {
	\'cid\':\'1\',
	\'pid\':\'5\'
}
headers = {
    \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36\',
}
options = {
    \'page-size\': \'A4\',
    \'margin-top\': \'0mm\',
    \'margin-right\': \'0mm\',
    \'margin-bottom\': \'0mm\',
    \'margin-left\': \'0mm\',
    # \'orientation\':\'Landscape\',#横向
    \'encoding\': "UTF-8",
    \'no-outline\': None,
      # \'footer-right\':\'[page]\' 设置页码
}
myMap=["UniversityPsychology","HigherEducationRegulations","HigherEducation","TeacherEthics"]
confg = pdfkit.configuration(wkhtmltopdf=\'/usr/local/bin/wkhtmltopdf\')
for i in range(4):
	for j in range(20):
		data[\'cid\'] = str(i+1)
		data[\'pid\'] = str(j+1)
		url=\'http://zjzx.zjnu.edu.cn/test/Default.aspx?cid=\'+str(i+1)+\'&pid=\'+str(j+1)
		#response = requests.post(url=url,data=data,headers=headers)
		myStr = myMap[i]+\'_\'+\'exam\'+str(j+1)+\'.pdf\'
		#response.encoding = \'utf-8\'
		myStr1 = myMap[i]+\'_\'+\'exam\'+str(j+1)+\'.html\'

		data[\'Button1\'] = \'提交并查看单选题答案\'
		data[\'drop1\']= \'1\'
		data[\'drop2\']=\'1\'
		data[\'__VIEWSTATE\'],data[\'__EVENTVALIDATION\'] = get_hiddenvalue(url) # 请查阅__VIEWSTATE __EVENTVALIDATION 一个是把所有的按键信息编码上传，一个是加密用的
		response = requests.post(url=url,data=data,headers=headers)
		responseReturn = response.text
		time.sleep(3)
		del data[\'Button1\']
		data[\'__VIEWSTATE\'],data[\'__EVENTVALIDATION\']=get_hiddenvalue_string(responseReturn)
		data[\'Button2\']=\'提交并查看多选题答案\'
		response = requests.post(url=url,data=data,headers=headers)
		responseReturn = response.text

		time.sleep(3)
		del data[\'Button2\']
		data[\'__VIEWSTATE\'],data[\'__EVENTVALIDATION\']=get_hiddenvalue_string(responseReturn)
		data[\'Button3\']=\'提交并查看判断题答案\'
		response = requests.post(url=url,data=data,headers=headers)
		responseReturn = response.text
		del data[\'Button3\']
		with open(myStr1,\'a\') as file:
			file.write(responseReturn)
				
		with open(myStr1, \'r\') as file:
			answer = file.read()
			pdfkit.from_string(answer, myStr,configuration=confg,options=options)
		
		#pdfkit.from_url(url, myStr,configuration=confg)