从抓取到放置在数据框中的不同长度

Question

为了在 facebook 页面中抓取帖子的反应，我可以抓取所有信息（cmets、反应、标签等），但是当我想将它们放入数据框中时，会出现错误（数组必须全部长度相同）这是正常的，因为有时有人只发表评论而另一个人只发表标签，所以我有不同长度的列表。我想我可以提出一个条件如果但可能有另一个优化的解决方案...... 例如 len(tag) =2, len(usr) = 17, len(commentaire)=12。

谢谢:)

        #imports here
    from selenium import webdriver
    from selenium.webdriver.common.keys import Keys
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support.wait import WebDriverWait
    import requests
    from bs4 import BeautifulSoup
    import time
    from time import sleep
    from lxml import html
    import logging as log
    import pandas as pd
    
    #chemin de chrome et desactivation des adds automatique de FB anti scrape
    chrome_options = webdriver.ChromeOptions()
    prefs = {"profile.default_content_setting_values.notifications" : 2}
    chrome_options.add_experimental_option("prefs",prefs)
    driver = webdriver.Chrome('C:/Users/User/Downloads/chromedriver.exe', 
    chrome_options=chrome_options)
    #open FB
    driver.get("http://www.facebook.com")
    print ("facebook page log ok")
    sleep(1)
    #reperage de user et pass  (css_selector)
    username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 
    "input[name='email']")))
    password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 
    "input[name='pass']")))
    
    ##reperage de user et pass et click  (xpath)
    #username = driver.find_element(By.XPATH,"//input[contains(@id,'email')]")
    #password = driver.find_element(By.XPATH,"//input[contains(@id,'pass')]")
    usr=input('Enter Email Id:') 
    pwd=input('Enter Password:') 
    #enter données
    username.clear()
    username.send_keys(usr)
    print ("Email Id entered")
    sleep(1)
    password.clear()
    password.send_keys(pwd)
    print ("Pass entered")
    #reperage bouton log in et click
    button = WebDriverWait(driver, 2).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 
    "button[type='submit']"))).click()
    print("login Successfully...")
    time.sleep(5)
    post = 'https://mbasic.facebook.com/AJSTunisie/posts/6452145678144265'
    #open the webpage
    driver.get(post)
    page = requests.get(post)
    df_comm = pd.DataFrame(columns = ['post_url', 'user', 'commentaire', 'tag', 'user_url'])
    page_count = 0
    while (True ):
    #scrape les reactions
            tree = html.fromstring(driver.page_source)
            user = tree.xpath("//div[@class='eb']/div/h3/a/text()")
            commentaire = tree.xpath("//div[@class='eb']/div/div[1]/text()")
            tag = tree.xpath("//div[@class='eb']/div/div[1]/a/text()")
            user_url = tree.xpath("//div[@class='eb']/div/h3/a/@href")
            data= {'post_url':[post]*len(user), 'user':user, 'commentaire':commentaire, 'tag':tag, 
            'user_url':user_url}
            df_comm = df_comm.append(pd.DataFrame(columns = df_comm.columns,data=data))
            #Check if more reaction exist ("En afficher davantage" existe ou pas)
            next_link = tree.xpath("//div[@class='eb eu']/a/@href")
            if len(next_link)!= 0:
                    driver.find_element_by_xpath("//div[@class='eb eu']/a/@href").click()
                    page_count = page_count+1
            else :
                    next_link = ''
                    break
    df_comm =df_comm.reset_index()
    #df_comm.to_csv(path,index=False)
    driver.close()

Answer 1

你应该以不同的方式来做。

首先，您应该找到所有 cmets - 带有文本、用户、标签等的元素 - 然后您应该使用 for-loop 分别处理每个评论。如果循环，您应该使用 relavite xpath（从 . 开始）仅获取此单个评论的信息。然后您可以查看是否缺少tag 或其他项目，您可以输入一些默认值 - 即空字符串。

这样每条评论都将包含所有值，因此 CSV 中的每一行都将具有相同的大小。

这种方式也解决了其他问题——在之前的方法中，你可以从第二条评论中获得带有标签的第一条评论，而你无法控制它。

为了使代码更简单，我将每个注释都放在行列表中，然后我将所有注释都转换为 DataFrame。

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from bs4 import BeautifulSoup
import time
from lxml import html
import logging as log
import pandas as pd

#chemin de chrome et desactivation des adds automatique de FB anti scrape
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs",prefs)
#driver = webdriver.Chrome('C:/Users/User/Downloads/chromedriver.exe', chrome_options=chrome_options)
driver = webdriver.Chrome(chrome_options=chrome_options)

#open FB
driver.get("http://www.facebook.com")
print ("facebook page log ok")
time.sleep(1)

#reperage de user et pass  (css_selector)
username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 
"input[name='email']")))
password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 
"input[name='pass']")))

##reperage de user et pass et click  (xpath)
#username = driver.find_element(By.XPATH,"//input[contains(@id,'email')]")
#password = driver.find_element(By.XPATH,"//input[contains(@id,'pass')]")
usr = input('Enter Email Id:') 
pwd = input('Enter Password:')

#enter données
username.clear()
username.send_keys(usr)
print ("Email Id entered")

#time.sleep(1)
password.clear()
password.send_keys(pwd)
print ("Pass entered")

#reperage bouton log in et click
button = WebDriverWait(driver, 2).until(EC.element_to_be_clickable((By.CSS_SELECTOR, 
"button[type='submit']"))).click()
print("login Successfully...")

time.sleep(5)

post_url = 'https://mbasic.facebook.com/AJSTunisie/posts/6452145678144265'
#open the webpage
driver.get(post_url)

all_rows = []
page_count = 0
while True:
#scrape les reactions
    page_count += 1
    print('\n--- page:', page_count, '---\n')
    
    tree = html.fromstring(driver.page_source)
    
    # find all comments
    all_comments = tree.xpath("//div[@class='ec']/div")
    print('len(all_comments):', len(all_comments))
    
    # work with every comment separatelly
    for comment in all_comments:
        
        user = comment.xpath(".//h3/a/text()")  # relative xpath starting at `.`
        print('user:', user)
        user = user[0] if user else ""  # set default value
        print('user:', user)
        
        commentaire = comment.xpath(".//div[1]/text()")  # relative xpath starting at `.`
        print('commentaire:', commentaire)
        commentaire = commentaire[0] if commentaire else ""  # set default value
        print('commentaire:', commentaire)
        
        tag = comment.xpath(".//div[1]/a/text()")  # relative xpath starting at `.` 
        print('tag:', tag)
        tag = tag[0] if tag else ""  # set default value
        print('tag:', tag)
        
        user_url = comment.xpath(".//h3/a/@href")  # relative xpath starting at `.`
        print('user_url:', user_url)
        user_url = user_url[0] if user_url else ""  # set default value
        print('user_url:', user_url)
                
        all_rows.append([post_url, user, commentaire, tag, user_url])
        
    #Check if more reaction exist ("En afficher davantage" existe ou pas)
    next_link = driver.find_elements_by_xpath("//div[@class='ec es']/a")
    print('---')
    print('len(next_link):', len(next_link))
    
    if next_link:        
        next_link[0].click()
        time.sleep(2)
    else:
        break

# - after loop - 

df = pd.DataFrame(all_rows, columns=['post_url', 'user', 'commentaire', 'tag', 'user_url'])
print(df)
df.to_csv('output.csv', index=False)
#driver.close()