仅获取具有特定月份和年份的值

Question

我有一个 HTML 列表，它包含一个表格。我想把所有的列写成一个列表。

最新数据始终标有<tr class="">。但是，我并不总是想访问最新的数据，而只想访问某些数据。 如果你看一下网站，你可以看到每个月都有数据。

我现在想说的是，我想获得 2021 年 8 月的数据。 我现在有以下问题：每个月有七个文件。前 5 个标有日、月和年。但是，最后两个标记为N/A，但仍属于同一天/月。

如何获取 2021 年 8 月的所有信息？

import requests
import re
from bs4 import BeautifulSoup
from datetime import datetime


DATASET_URL = "http://insideairbnb.com/get-the-data.html"
DATASET_CITY = "Antwerp"
DATASET_MONTHYEAR = "09.2021"

# Converts 29 September, 2021 to 09.2021
def datetimeConverter(d):
    if(d == 'N/A'):
        return 'N/A'
    return datetime.strptime(d, '%m.%Y').strftime('%d %B, %Y')

#use requests
r = requests.get(DATASET_URL)
content = r.content

#soup!
soup = BeautifulSoup(content, "html.parser")


city_table = soup.find(class_=DATASET_CITY.lower())

print(city_table)

table class="table table-hover table-striped antwerp">
<thead>
<tr>
<th class="col-md-3" data-field="host_id">Date Compiled</th>
<th class="col-md-3" data-field="host_id">Country/City</th>
<th class="col-md-3" data-field="host_id">File Name</th>
<th class="col-md-3" data-align="right" data-field="count">
                        Description
                    </th>
</tr>
</thead>
<tbody>
<tr class="">
<td>29 September, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/data/listings.csv.gz" onclick="var that=this;ga('send','event', 'download','listings',this.href);setTimeout(function(){location.href=that.href;},200);return false;">listings.csv.gz</a></td>
<td>Detailed Listings data for Antwerp</td>
</tr>
<tr class="">
<td>29 September, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/data/calendar.csv.gz" onclick="var that=this;ga('send','event', 'download','calendar',this.href);setTimeout(function(){location.href=that.href;},200);return false;">calendar.csv.gz</a></td>
<td>Detailed Calendar Data for listings in Antwerp</td>
</tr>
<tr class="">
<td>29 September, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/data/reviews.csv.gz" onclick="var that=this;ga('send','event', 'download','reviews',this.href);setTimeout(function(){location.href=that.href;},200);return false;">reviews.csv.gz</a></td>
<td>Detailed Review Data for listings in Antwerp</td>
</tr>
<tr class="">
<td>29 September, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/visualisations/listings.csv" onclick="var that=this;ga('send','event', 'download','listings_visualisation',this.href);setTimeout(function(){location.href=that.href;},200);return false;">listings.csv</a></td>
<td>Summary information and metrics for listings in Antwerp (good for visualisations).</td>
</tr>
<tr class="">
<td>29 September, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/visualisations/reviews.csv" onclick="var that=this;ga('send','event', 'download','reviews_visualisation',this.href);setTimeout(function(){location.href=that.href;},200);return false;"> reviews.csv</a></td>
<td>Summary Review data and Listing ID (to facilitate time based analytics and visualisations linked to a listing).</td>
</tr>
<tr class="">
<td>N/A</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/visualisations/neighbourhoods.csv" onclick="var that=this;ga('send','event', 'download','neighbourhoods',this.href);setTimeout(function(){location.href=that.href;},200);return false;">neighbourhoods.csv</a></td>
<td>Neighbourhood list for geo filter. Sourced from city or open source GIS files.</td>
</tr>
<tr class="">
<td>N/A</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/visualisations/neighbourhoods.geojson" onclick="var that=this;ga('send','event', 'download','geojson',this.href);setTimeout(function(){location.href=that.href;},200);return false;">neighbourhoods.geojson</a></td>
<td>GeoJSON file of neighbourhoods of the city.</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/data/listings.csv.gz" onclick="var that=this;ga('send','event', 'download','listings',this.href);setTimeout(function(){location.href=that.href;},200);return false;">listings.csv.gz</a></td>
<td>Detailed Listings data for Antwerp</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/data/calendar.csv.gz" onclick="var that=this;ga('send','event', 'download','calendar',this.href);setTimeout(function(){location.href=that.href;},200);return false;">calendar.csv.gz</a></td>
<td>Detailed Calendar Data for listings in Antwerp</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/data/reviews.csv.gz" onclick="var that=this;ga('send','event', 'download','reviews',this.href);setTimeout(function(){location.href=that.href;},200);return false;">reviews.csv.gz</a></td>
<td>Detailed Review Data for listings in Antwerp</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/listings.csv" onclick="var that=this;ga('send','event', 'download','listings_visualisation',this.href);setTimeout(function(){location.href=that.href;},200);return false;">listings.csv</a></td>
<td>Summary information and metrics for listings in Antwerp (good for visualisations).</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/reviews.csv" onclick="var that=this;ga('send','event', 'download','reviews_visualisation',this.href);setTimeout(function(){location.href=that.href;},200);return false;"> reviews.csv</a></td>
<td>Summary Review data and Listing ID (to facilitate time based analytics and visualisations linked to a listing).</td>
</tr>
<tr class="archived">
<td>N/A</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/neighbourhoods.csv" onclick="var that=this;ga('send','event', 'download','neighbourhoods',this.href);setTimeout(function(){location.href=that.href;},200);return false;">neighbourhoods.csv</a></td>
<td>Neighbourhood list for geo filter. Sourced from city or open source GIS files.</td>
</tr>
<tr class="archived">
<td>N/A</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/neighbourhoods.geojson" onclick="var that=this;ga('send','event', 'download','geojson',this.href);setTimeout(function(){location.href=that.href;},200);return false;">neighbourhoods.geojson</a></td>
<td>GeoJSON file of neighbourhoods of the city.</td>
</tr>

我想要什么

enter image description here

列表应该是最后的样子

list= [["http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/data/listings.csv.gz", 
         "listings.csv.gz",
         "Description", "27 August, 2021"]
         ,[...],
        ["http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/neighbourhoods.geojson", 
         "neighbourhoods.geojson",
         "Description", "27 August, 2021"]]

我想要的html

<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/data/listings.csv.gz" onclick="var that=this;ga('send','event', 'download','listings',this.href);setTimeout(function(){location.href=that.href;},200);return false;">listings.csv.gz</a></td>
<td>Detailed Listings data for Antwerp</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/data/calendar.csv.gz" onclick="var that=this;ga('send','event', 'download','calendar',this.href);setTimeout(function(){location.href=that.href;},200);return false;">calendar.csv.gz</a></td>
<td>Detailed Calendar Data for listings in Antwerp</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/data/reviews.csv.gz" onclick="var that=this;ga('send','event', 'download','reviews',this.href);setTimeout(function(){location.href=that.href;},200);return false;">reviews.csv.gz</a></td>
<td>Detailed Review Data for listings in Antwerp</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/listings.csv" onclick="var that=this;ga('send','event', 'download','listings_visualisation',this.href);setTimeout(function(){location.href=that.href;},200);return false;">listings.csv</a></td>
<td>Summary information and metrics for listings in Antwerp (good for visualisations).</td>
</tr>
<tr class="archived">
<td>27 August, 2021</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/reviews.csv" onclick="var that=this;ga('send','event', 'download','reviews_visualisation',this.href);setTimeout(function(){location.href=that.href;},200);return false;"> reviews.csv</a></td>
<td>Summary Review data and Listing ID (to facilitate time based analytics and visualisations linked to a listing).</td>
</tr>
<tr class="archived">
<td>N/A</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/neighbourhoods.csv" onclick="var that=this;ga('send','event', 'download','neighbourhoods',this.href);setTimeout(function(){location.href=that.href;},200);return false;">neighbourhoods.csv</a></td>
<td>Neighbourhood list for geo filter. Sourced from city or open source GIS files.</td>
</tr>
<tr class="archived">
<td>N/A</td>
<td>Antwerp</td>
<td><a href="http://data.insideairbnb.com/belgium/vlg/antwerp/2021-08-27/visualisations/neighbourhoods.geojson" onclick="var that=this;ga('send','event', 'download','geojson',this.href);setTimeout(function(){location.href=that.href;},200);return false;">neighbourhoods.geojson</a></td>
<td>GeoJSON file of neighbourhoods of the city.</td>
</tr>

Answer 1

您必须进一步过滤结果以获得所需的输出。

下面的代码应该会得到你想要的结果：

import requests
from bs4 import BeautifulSoup
from datetime import datetime

DATASET_URL = "http://insideairbnb.com/get-the-data.html"
DATASET_CITY = "Antwerp"
DATASET_MONTHYEAR = "09.2021"

# Converts 09.2021 to 2021-09
def datetimeConverter(d):
    return datetime.strptime(d, '%m.%Y').strftime('%Y-%m')


def filter_records_by_date(record):
    return datetimeConverter(DATASET_MONTHYEAR) in record.find_all('td')[2].a.get('href')


# use requests
r = requests.get(DATASET_URL)
content = r.content

# soup!
soup = BeautifulSoup(content, "html.parser")
city_table = soup.find(class_=DATASET_CITY.lower()).findAll('tr')[1:]
filtered_city_table = list(filter(filter_records_by_date, city_table))
query_date = filtered_city_table[0].td.text
data = []
for row in filtered_city_table:
    cells = row.findChildren('td')
    val = [cells[2].find('a').get("href"), cells[3].text, query_date]
    data.append(val)
print(data)
print(len(data))

输出： [ ['http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/data/listings.csv.gz', 'Detailed Listings data for Antwerp', '29 September, 2021'], ['http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/data/calendar.csv.gz', 'Detailed Calendar Data for listings in Antwerp', '29 September, 2021'], ['http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/data/reviews.csv.gz', 'Detailed Review Data for listings in Antwerp', '29 September, 2021'], ['http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/visualisations/listings.csv', 'Summary information and metrics for listings in Antwerp (good for visualisations).', '29 September, 2021'], ['http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/visualisations/reviews.csv', 'Summary Review data and Listing ID (to facilitate time based analytics and visualisations linked to a listing).', '29 September, 2021'], ['http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/visualisations/neighbourhoods.csv', 'Neighbourhood list for geo filter. Sourced from city or open source GIS files.', '29 September, 2021'], ['http://data.insideairbnb.com/belgium/vlg/antwerp/2021-09-29/visualisations/neighbourhoods.geojson', 'GeoJSON file of neighbourhoods of the city.', '29 September, 2021']]