utils.py

from random import choice

with open('user-agents.txt') as f:
    USER_AGENTS = [line.strip() for line in f]


class BColors:
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    ENDC = '\033[0m'


def print_warn(txt):
    print(f'{BColors.WARNING}{txt}{BColors.ENDC}')


def print_green(txt):
    print(f'{BColors.OKGREEN}{txt}{BColors.ENDC}')


def batch(iterable, size):
    """Returns elements from iterator in batches of size"""
    iterator = iter(iterable)
    while True:
        res = []
        for _ in range(size):
            try:
                res.append(next(iterator))
            except StopIteration:
                break

        if not res:
            return

        yield res


async def fetch(session, url):
    headers = {'User-Agent': choice(USER_AGENTS)}
    async with session.get(url, headers=headers) as response:
        content = await response.read()
        return response.status, content