打印网络爬虫输出

Question

我想使用 Flask 在 Python 中为网络爬虫 I wrote 创建一个网络界面。我无法打印结果，所以想改为显示一个列表。如何在login.html 页面中打印for 循环的结果？

from flask import Flask, render_template, redirect, url_for, request

from bs4 import BeautifulSoup, SoupStrainer

import urllib2

import re

from flask import jsonify

app = Flask(__name__)

@app.route('/login', methods=['GET', 'POST'])

def login():
    url = "example.com"

    url_list = ["example.com/1", "example.com/2"]
    found_list = []

    if request.method == 'POST':
        if request.form['inpur_url'] != 'example.com':
            error = 'Invalid Credentials. Please try again.'
        else:
            for line1 in url_list:
                 #print "Crawled" " " + line1
                 try:
                     html_page = urllib2.urlopen(line1)
                     soup = BeautifulSoup(html_page)
                     link = soup.findAll(href=True)
                 except urllib2.HTTPError:
                   pass
                 for link1 in link:
                     url1 = link1.get("href")
                     if url in url1:
                         found_list.append(url)
                 return jsonify(found_list)     

    #return render_template('login.html', error=error)
    return jsonify(found_list)     

if __name__ == '__main__':
    app.run(debug=True)

Answer 1

我不确定您的整体设计决策，但我很确定您的函数中的逻辑实际上不会在网页中找到链接。这将利用 urllib2 和 BeautifulSoup 为您获取链接列表：

if request.method == 'POST':
    if request.form['inpur_url'] != 'example.com':
        error = 'Invalid Credentials. Please try again.'
    else:
        for url in url_list:
            try:
                handle = urllib2.urlopen(url)
                contents = handle.read()
                soup = BeautifulSoup(contents)
                links = soup.find_all("a")
                for link in links:
                    found_list.append(link.get("href"))
            except urllib2.HTTPError:
                pass