当在字符串中找不到子字符串时，然后解析网站

Question

这是我的基本代码：

import urllib
import webbrowser
from bs4 import BeautifulSoup
import requests

address = 'https://google.com/search?q='
# Default Google search address start
file = open( "OCR.txt", "rt" )
# Open text document that contains the question
word = file.read()
file.close()

myList = [item for item in word.split('\n')]
newString = ' '.join(myList)
# The question is on multiple lines so this joins them together with proper spacing

qstr = urllib.parse.quote_plus(newString)
# Encode the string

newWord = address + qstr
# Combine the base and the encoded query

response = requests.get(newWord)

#with open('output.html', 'wb') as f:
#    f.write(response.content)
#webbrowser.open('output.html')

answers = open("ocr2.txt", "rt")

ansTable = answers.read()
answers.close()

ans = ansTable.splitlines()

ans1 = str(ans[0])
ans2 = str(ans[2])
ans3 = str(ans[4])

print(' ')
print(newWord)
print(' ')
print(newString)
print(' ')
print("1. "+ans1)
print("2. "+ans2)
print("3. "+ans3)

ans1Score = 0
ans2Score = 0
ans3Score = 0

soup = BeautifulSoup(response.text, 'lxml')
for g in soup.find_all(class_='g'):

    webBlock = str(g)

    ans1Tally = webBlock.count(ans1)
    ans2Tally = webBlock.count(ans2)
    ans3Tally = webBlock.count(ans3)

    if  webBlock.find(ans1)!=-1:

        ans1Score += ans1Tally

    if webBlock.find(ans2)!=-1:

        ans2Score += ans2Tally

    if webBlock.find(ans3)!=-1:

        ans3Score += ans3Tally

    #print(ans1+": "+str(ans1Score))
    #print(ans2+": "+str(ans2Score))
    #print(ans3+": "+str(ans3Score))
    #print('-----')
    #print(g.text)

print(' ')
print('-----')
print(ans1+": "+str(ans1Score))
print(ans2+": "+str(ans2Score))
print(ans3+": "+str(ans3Score))
print('-----')

#print(g.text)

我基本上想要实现的是，如果在文本块中找不到该短语（这是一个谷歌搜索结果，所以它包含标题、链接和文本的小摘要。如果我打印 webBlock 这个是它包含的一个例子：

<div class="g"><h3 class="r"><a href="/url?q=https://www.rollingstone.com/glixel/news/lists/bloodbowl-and-beyond-the-most-absurdly-violent-football-games-of-all-time-w464896/mutant-league-football-w464904&amp;sa=U&amp;ved=0ahUKEwjSxdiFq5_YAhWlSd8KHSX-DtUQFghKMAk&amp;usg=AOvVaw2e_Z2FBOjUfryPIGK3yhEq"><b>Mutant League</b> Football | 'Bloodbowl' and Beyond: The Most ...</a></h3><div class="s"><div class="kv" style="margin-bottom:2px"><cite>https://www.rollingstone.com/...<b>games</b>.../<b>mutant-league</b>-football-w464904</cite><div class="_nBb"><div aria-expanded="false" aria-haspopup="true" data-ved="0ahUKEwjSxdiFq5_YAhWlSd8KHSX-DtUQ7B0ISzAJ" onclick="google.sham(this);" style="display:inline" tabindex="0"><span class="_O0"></span></div><div class="am-dropdown-menu" role="menu" style="display:none" tabindex="-1"><ul><li class="_Ykb"><a class="_Zkb" href="/url?q=http://webcache.googleusercontent.com/search%3Fq%3Dcache:7JwZzaQGVBsJ:https://www.rollingstone.com/glixel/news/lists/bloodbowl-and-beyond-the-most-absurdly-violent-football-games-of-all-time-w464896/mutant-league-football-w464904%252BThe%2B%2522Mutant%2BLeague%2522%2Bvideo%2Bgames%2Bdid%2BNOT%2Binclude%2Bwhich%2Bsport%3F%26hl%3Den%26ct%3Dclnk&amp;sa=U&amp;ved=0ahUKEwjSxdiFq5_YAhWlSd8KHSX-DtUQIAhNMAk&amp;usg=AOvVaw1l8NtEdw1gvsqGLNw13Ulf">Cached</a></li></ul></div></div></div><span class="st">Feb 3, 2017 <b>...</b> In which Electronic Arts realized they still had the durable Madden '93 engine <br/>
lying around, and decide to commit their talents to more than one football <b>game</b> <br/>
every year. <b>Mutant League</b> Football for the Sega Genesis is ridiculous . It takes <br/>
place in a post-apocalyptic wasteland where radiation has mottled ...</span><br/></div></div>

我想要做的是，如果该文本块不包含字符串，我想从中提取链接，然后执行我对原始 google 搜索链接所做的操作：

解析数据以收集所有标签，然后扫描以查看短语是否在其中，然后将其添加到计数中。它必须对所有短语都这样做。

我遇到的一些问题是：

并非所有链接都包含 http://、https:// 或 www.。例如 mario.wikia.com。不管它是什么，我都希望能够拉出链接。

我已经在这个问题上绞尽脑汁一段时间了，所以我想我会把它扔在这里，看看是否有人有任何解决方案。我确信可能有一个简单的答案，但我是 Python 新手，并且有很多“doh”时刻。

无论如何，感谢您的帮助并感谢您的阅读！

Answer 1

g 是 soup 的类似对象，因此您可以使用相同的函数从 a 获取 url

 g.find('a')['href']

完整的网址将是

 url = 'https://google.com' + g.find('a')['href']

代码：

import requests
from bs4 import BeautifulSoup
import urllib

def do_something_with_page(url):
    response = requests.get(full_url)

    soup = BeautifulSoup(response.text, 'lxml')

    print("... DO SOMETHING WITH THIS PAGE ...")
    print()


# ---------------------------------------------------

address = 'https://google.com/search?q='

# --- files ---

with open("ocr.txt") as f:
    newString = f.read()
newString = newString.replace("\n", " ").strip()

with open("ocr2.txt") as f:
    ans = f.read().splitlines()
ans1 = str(ans[0])
ans2 = str(ans[2])
ans3 = str(ans[4])

# --- info ---

quote_str = urllib.parse.quote_plus(newString)
full_url = address + quote_str

print()
print('full url:', full_url)
print()
print('newString:', newString)
print()
print("1.", ans1)
print("2.", ans2)
print("3.", ans3)
print()

# --- request ---

response = requests.get(full_url)

soup = BeautifulSoup(response.text, 'lxml')

for g in soup.find_all(class_='g'):

    url = 'https://google.com' + g.find('a')['href']
    print(url, '\n')

    webBlock = g.text # text without tags
    #webBlock = str(g)

    count1 = webBlock.count(ans1)
    count2 = webBlock.count(ans2)
    count3 = webBlock.count(ans3)

    ans1Score += count1
    ans2Score += count2
    ans3Score += count3


    if count1 + count2 + count3 == 0:
        do_something_with_page(url)


# --- results ---

print()
print('-----')
print(ans1, ":", ans1Score)
print(ans2, ":", ans2Score)
print(ans3, ":", ans3Score)
print('-----')