David Durst
02/23/2022, 9:33 AMDavid Durst
02/23/2022, 9:34 AMDavid Durst
02/23/2022, 9:35 AMdef get_category_items_segment(segment):
session = create_new_oauth_session()
resp = session.get(url, params={"category": segment[0], "start": segment[1], "end": segment[2]}, headers={'Content-Type': 'application/json'})
result = resp.json()
if len(result["items"]) == 0:
print("Empty return for: " + str(segment[0]) + " start: "+str(segment[1])+" end: "+str(segment[2]))
return result["items"]
def create_segment_groups(total, segment_size, data):
segments = []
start = 0
end = segment_size
while total > start:
if end > total:
end = total
segments.append([data, start, end])
start += segment_size
end += segment_size
return segments
def get_category_items(category):
items = []
segments = create_segment_groups(category["item_count"], 50, category["id"])
with Pool(processes=10) as p:
item_lists = p.map(func=get_category_items_segment, iterable=segments)
for item_list in item_lists:
items.extend(item_list)
return items
David Durst
02/23/2022, 9:35 AMDavid Durst
02/23/2022, 9:37 AM