Instaloader JSON 文件：将 200 个 JSON 文件转换为单个 CSV (Python 3.7)

Question

我想使用 Instaloader 从特定的 Instagram Hashtag（例如#moodoftheday）自动下载图片（或视频）及其标题和其他数据。 Instaloader 返回包​​含帖子元数据的 JSON 文件。

以下代码仅使用单个 @user_profile 元数据。 我也想做同样的事情，但对于#hashtag 不是特定的@user。

最终目标是将所有 JSON 文件（例如 200 个）放入 csv 文件中。 如何在干净的 excel/CSV 文件中处理我下载的数据？

这是我的代码：

# Install Instaloader
import instaloader


def get_instagram_posts(username, startdate, enddate):
   # Create an instaloader object with parameters
   L = instaloader.Instaloader(download_pictures = False, download_videos = False, download_comments= False, compress_json = False)
   
   # Log in with the instaloader object
   L.login("username" , "password")
   # Search the instagram profile
   profile = instaloader.Profile.from_username(L.context, username) 
   # Scrape the posts
   posts = profile.get_posts()
   for post in takewhile(lambda p: p.date > startdate, dropwhile(lambda p : p.date > enddate, posts)): 
   print(post.date)
   L.download_post(post, target = profile.username)

'''
This function will now save all instagram posts and related data to a folder in you current working directory.
Let’s call this function on the instagram account of “moodoftheday”. let the script do its magic.
This might take a while so be patient. 
'''
  
import os
import datetime
# instagram username
username = "realdonaldtrump"
# daterange of scraping
startdate = datetime(2020, 9, 1)
enddate = datetime(2020, 10, 1)
# get your current working directory
current_wkdir = os.get_cwd()
# Call the function. This will automatically store all the scrape data in a folder in your current working directory
get_instagram_posts(username, startdate, enddate)


'''
You notice that this data is NOT yet in the right format since each post has a separate json file. 
You will need to process all these json files to a consolidated excel file in order to perform analyses on the data.
'''


def parse_instafiles(username, path):
    """ 
    This function loads in all the json files generated by the instaloader package and parses it into a csv file.
    """
    #print('Entering provided directory...')
    os.chdir(os.path.join(path, username))
    
    columns = ['filename', 'datetime', 'type', 'locations_id', 'locations_name', 'mentions', 'hashtags', 'video_duration']
    
    dataframe = pd.DataFrame(columns=[])
    
    #print('Traversing file tree...')
    
    glob('*UTC.json')
    
    for file in glob('*UTC.json'):
        with open(file, 'r') as filecontent:
            filename = filecontent.name
            #print('Found JSON file: ' + filename + '. Loading...')
            
            try:
                metadata = orjson.loads(filecontent.read())
            
            except IOError as e:
                #print("I/O Error. Couldn't load file. Trying the next one...")
                continue
            else:
                pass
            #print('Collecting relevant metadata...')
            time = datetime.fromtimestamp(int(metadata['node']['taken_at_timestamp']))
            type_ = metadata['node']['__typename']
            likes = metadata['node']['edge_media_preview_like']['count']     
            comments = metadata['node']['edge_media_to_comment']['count']
            username = metadata['node']['owner']['username']
            followers = metadata['node']['owner']['edge_followed_by']['count']
            try:
                text = metadata['node']['edge_media_to_caption']['edges'][0]['node']['text']
            except:
                text = ""
            try:
                post_id = metadata['node']['id']
            except:
                post_id = ""
            minedata = {'filename': filename, 'time': time, 'text': text,
                    'likes': likes, 'comments' : comments, 'username' : username,  'followers' : followers, 'post_id' : post_id}
            #print('Writing to dataframe...')
            dataframe = dataframe.append(minedata, ignore_index=True)
            #print('Closing file...')
            del metadata
            filecontent.close()
    #print('Storing dataframe to CSV file...')
    #print('Done.')
    dataframe['source'] = 'Instagram'
    return dataframe

'''
You can then use this function to process the "moodoftheday" Instagram data.
'''  

df_instagram = parse_instafiles(username, os.getcwd() )
df_instagram.to_excel("moodoftheday.csv")

我对 Python 和整体编程非常陌生，因此非常感谢任何帮助！ 先感谢您！索非亚

Answer 1

Instaloader 的文档中有一个标签搜索示例，代码如下：

from datetime import datetime
import instaloader

L = instaloader.Instaloader()

posts = instaloader.Hashtag.from_name(L.context, "urbanphotography").get_posts()

SINCE = datetime(2020, 5, 10)  # further from today, inclusive
UNTIL = datetime(2020, 5, 11)  # closer to today, not inclusive

k = 0  # initiate k
#k_list = []  # uncomment this to tune k

for post in posts:
    postdate = post.date

    if postdate > UNTIL:
        continue
    elif postdate <= SINCE:
        k += 1
        if k == 50:
            break
        else:
            continue
    else:
        L.download_post(post, "#urbanphotography")
        # if you want to tune k, uncomment below to get your k max
        #k_list.append(k)
        k = 0  # set k to 0

#max(k_list)

这里是更多信息的链接：

https://instaloader.github.io/codesnippets.html

我正在尝试做类似的事情，但我对编程还是很陌生，所以如果我不能提供太多帮助，我很抱歉

Answer 2

我做了一些更改，它没有显示错误，但仍然需要一些专业的工作：

import instaloader
from datetime import datetime
import datetime
from itertools import takewhile
from itertools import dropwhile
import os
import glob as glob
import json
import pandas as pd
import csv

lusername = ''
lpassword = ''

def get_instagram_posts(username, startdate, enddate):
   # Create an instaloader object with parameters
   L = instaloader.Instaloader(download_pictures = False, download_videos = False, download_comments= False, compress_json = False)
   
   # Log in with the instaloader object
   L.login("lusername" , "lpassword")

   # Search the instagram profile
   profile = instaloader.Profile.from_username(L.context, username) 
   # Scrape the posts
   posts = profile.get_posts()
   for post in takewhile(lambda p: p.date > startdate, dropwhile(lambda p : p.date > enddate, posts)): 
    print(post.date)
   L.download_post(post, target = profile.username)


# instagram username
username = "realdonaldtrump"
# daterange of scraping
startdate = datetime.datetime(2020, 9, 1,0,0)
enddate = datetime.datetime(2022, 2, 1,0,0)
# get your current working directory
current_wkdir = os.getcwd()
# Call the function. This will automatically store all the scrape data in a folder in your current working directory
get_instagram_posts(username, startdate, enddate)


def parse_instafiles(username, path):
    #print('Entering provided directory...')
    os.chdir(os.path.join(path, username))
    
    columns = ['filename', 'datetime', 'type', 'locations_id', 'locations_name', 'mentions', 'hashtags', 'video_duration']
    
    dataframe = pd.DataFrame(columns=[])
    
    #print('Traversing file tree...')
    
#    glob('*UTC.json')
    
    for file in glob.glob('*UTC.json'):
        with open(file, 'r') as filecontent:
            filename = filecontent.name
            #print('Found JSON file: ' + filename + '. Loading...')
            
            try:
                metadata = json.load(filecontent)
            
            except IOError as e:
                #print("I/O Error. Couldn't load file. Trying the next one...")
                continue
            else:
                pass
            #print('Collecting relevant metadata...')
            time = datetime.datetime.fromtimestamp(int(metadata['node']['taken_at_timestamp']))
            type_ = metadata['node']['__typename']
            likes = metadata['node']['edge_media_preview_like']['count']     
            comments = metadata['node']['edge_media_to_comment']['count']
            username = metadata['node']['owner']['username']
            followers = metadata['node']['owner']['edge_followed_by']['count']
            try:
                text = metadata['node']['edge_media_to_caption']['edges'][0]['node']['text']
            except:
                text = ""
            try:
                post_id = metadata['node']['id']
            except:
                post_id = ""
            minedata = {'filename': filename, 'time': time, 'text': text,
                    'likes': likes, 'comments' : comments, 'username' : username,  'followers' : followers, 'post_id' : post_id}
            #print('Writing to dataframe...')
            dataframe = dataframe.append(minedata, ignore_index=True)
            #print('Closing file...')
            del metadata
            filecontent.close()
    #print('Storing dataframe to CSV file...')
    #print('Done.')
    dataframe['source'] = 'Instagram'
    return dataframe

'''
You can then use this function to process the "moodoftheday" Instagram data.
'''  

df_instagram = parse_instafiles(username, os.getcwd() )
df_instagram.to_csv("moodoftheday.csv")