arxivdf.reset_index(inplace=True) # didn't drop the index coz I will need it later
tmp = arxivdf.explode('data') # 'data' is the column that has a list of dictionaries
dict2df = tmp['data'].apply(pd.Series)[['category', 'link', 'summary', 'title']] # the column passed are keys,
arxivdf = pd.concat((tmp,dict2df),axis=1) # concatenate to have everything in on df