【发布时间】：2022-08-18 23:04:18
【问题描述】：
def scrape_wikipedia(name_topic, verbose=True):
def link_to_wikipedia(link):
try:
page = api_wikipedia.page(link)
if page.exists():
return {\'page\': link, \'text\': page.text, \'link\': page.fullurl, \'categories\': list(page.categories.keys())}
except:
return None
api_wikipedia = wikipediaapi.Wikipedia(language=\'en\', extract_format=wikipediaapi.ExtractFormat.WIKI)
name_of_page = api_wikipedia.page(name_topic)
if not name_of_page.exists():
print(\'Page {} is not present\'.format(name_of_page))
return
links_to_page = list(name_of_page.links.keys())
procceed = tqdm(desc=\'Scraped links\', unit=\'\', total=len(links_to_page)) if verbose else None
origin = [{\'page\': name_topic, \'text\': name_of_page.text, \'link\': name_of_page.fullurl, \'categories\': list(name_of_page.categories.keys())}]
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
links_future = {executor.submit(link_to_wikipedia, link): link for link in links_to_page}
for future in concurrent.futures.as_completed(links_future):
info = future.result()
origin.append(info) if info else None
procceed.update(1) if verbose else None
procceed.close() if verbose else None
namespaces = (\'Wikipedia\', \'Special\', \'Talk\', \'LyricWiki\', \'File\', \'MediaWiki\',
\'Template\', \'Help\', \'User\', \'Category talk\', \'Portal talk\')
origin = pds.DataFrame(origin)
origin = origin[(len(origin[\'text\']) > 20)
& ~(origin[\'page\'].str.startswith(namespaces, na=True))]
origin[\'categories\'] = origin.categories.apply(lambda a: [b[9:] for b in a])
origin[\'topic\'] = name_topic
print(\'Scraped pages\', len(origin))
return origin
SSLError: HTTPSConnectionPool(host=\'en.wikipedia.org\', port=443): url: /w/api.php?action=query&prop=info&titles=COVID+19&inprop=protection%7Ctalkid%7Cwatched% 7Cwatchers%7Cvisitingwatchers%7Cnotificationtimestamp%7Csubjectid%7Curl%7Creadable%7Cpreload%7Cdisplaytitle&format=json&redirects=1 (由 SSLError(SSLCertVerificationError(1, \'[SSL: CERTIFICATE_VERIFY_FAILED] 证书验证失败：无法获取本地颁发者证书(_ssl.c:第1129章）
标签： python machine-learning web-scraping jupyter-notebook data-science