如何在包含链接列表的表格中提取可下载链接，并抓取多个页面

Question

点击表格内的特定标题（即本例中的公告）时，我想提取所有 .doc 链接。

我可以按照以下代码为仅一个页面提取第一级中的标题、日期和所有链接：

from lxml import etree
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import sys
import pandas as pd
from urllib.request import urlparse, urljoin
from bs4 import BeautifulSoup
import requests

frame =[]

chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(options = chrome_options)

for page_number in range(1,78):
    url = 'http://example.com/index{}.html'.format(page_number)

driver.get(url)
html = etree.HTML(driver.page_source)

extract_announcements_list = html.xpath('//table[@id="14681"]/tbody/tr/td/table[@width="90%"][position()>=2 and position() <= (last())]')
for i in list:
    date = i.xpath('./tbody/tr/td[3]/text()')
    title = i.xpath('./tbody/tr/td[2]/font/a/@title')
    link = i.xpath('./tbody/tr/td[2]/font/a/@href')
    real_link = 'http://example.com'+ link[0]
    print(title,date,real_link)

frame.append({
    'title': title,
    'link': real_link,
    'date': date,
    **'content': doc_link,** #this is the doc_link I want to extract in the second level
        })

dfs = pd.DataFrame(frame)
dfs.to_csv('myscraper.csv',index=False,encoding='utf-8-sig')

我正在努力寻找解决方案。如果有人能帮我提取第二个链接以获取 .doc 链接的内容（'content': doc_link），以及抓取网站中所有页面的方法，我将不胜感激。

非常感谢您！

更新：非常感谢@Ares Zephyr 分享您的代码。这是我根据建议对代码所做的。但它并没有产生任何能够获得内部链接的结果。

from lxml import etree
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import sys
import pandas as pd
import urllib.request
from bs4 import BeautifulSoup
import requests

frame =[]

chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
driver = webdriver.Chrome(options = chrome_options)

for page_number in range(1,2):
    url = 'http://example.com/index{}.html'.format(page_number)
    print('Downloading page %s...' % url)

    driver.get(url)
    html = etree.HTML(driver.page_source)
    html_page = urllib.request.urlopen(url)
    soup = BeautifulSoup(html_page, "html.parser")

    extract_announcements_list = html.xpath('//table[@id="14681"]/tbody/tr/td/table[@width="90%"][position()>=2 and position() <= (last())]')
    for i in list:
       date = i.xpath('./tbody/tr/td[3]/text()')
       title = i.xpath('./tbody/tr/td[2]/font/a/@title')
       link = i.xpath('./tbody/tr/td[2]/font/a/@href')
       real_link = 'http://example.com'+ link[0]

          soup = BeautifulSoup(requests.get(real_link).content, 'html.parser')
          for doc_link in soup.findAll('a'):
             thelink = doc_link.get('href')

             frame.append({
               'title': title,
               'link': real_link,
               'date': date,
               'doclink': thelink,
               })

dfs = pd.DataFrame(frame)
dfs.to_csv('myscraper.csv',index=False,encoding='utf-8-sig')

Answer 1

您需要在代码中缩进这一段，以便 append 函数可以处理所有抓取的数据。我相信这也是@arundeep-chohan 想要强调的。

frame.append({
    'title': announcement_title,
    'link': real_link,
    'date': announcement_date,
    **'content': doc_link,** #this is the doc_link I want to extract in the second level
        })

查找doc文件的逻辑如下。请修改并使用它。这是我用来下载 pdf 文件的代码的一部分。

for link in soup.findAll('a'):
  theLink = link.get('href')
  name= link.string

  # Logic to find .pdf files
 
  if theLink[-4:]  == ".pdf" or theLink[-4:] == ".pdf":
    if theLink[-4:] ==".pdf":
      fileExtension = ".pdf"