5 Best Web Scraping Tools (Ranked By Performance, Stability & Price)

				
					import requests
import time
import random

# List of random words to use in the search query
search_terms_amazon = ["https://www.amazon.com/dp/B0CTKXMQXK", "https://www.amazon.com/dp/B0D1ZFS9GH", "https://www.amazon.com/dp/B0CXG3HMX1", "https://www.amazon.com/dp/B0CKM9JVJB/", "https://www.amazon.com/dp/B0BYX1XT81"]
search_terms_idealista = ['https://www.idealista.com/venta-viviendas/torrelavega/inmobiliaria-barreda/','https://www.idealista.com/venta-viviendas/torrelavega/inmobiliaria-barreda/','https://www.idealista.com/venta-viviendas/torrelavega/inmobiliaria-barreda/','https://www.idealista.com/venta-viviendas/torrelavega/inmobiliaria-barreda/','https://www.idealista.com/alquiler-viviendas/malaga-malaga/con-solo-pisos%2Caticos/']
search_terms_google = [
    "pizza", "burger", "sushi", "coffee", "tacos", "salad", "pasta", "steak",
    "sandwich", "noodles", "bbq", "dumplings", "shawarma", "falafel",
    "pancakes", "waffles", "curry", "soup", "kebab", "ramen"
  ];


# base_url = Your-API-URL

total_requests = 50
success_count = 0
total_time = 0

for i in range(total_requests):
    try:
        # Pick a random search term from the list
        search_term = random.choice(search_terms)
        url = base_url.format(query=search_term)

        start_time = time.time()  # Record the start time
        response = requests.get(url)
        end_time = time.time()  # Record the end time

        # Calculate the time taken for this request
        request_time = end_time - start_time
        total_time += request_time

        # Check if the request was successful (status code 200)
        if response.status_code == 200:
            success_count += 1
        print(f"Request {i+1} with search term '{search_term}' took {request_time:.2f} seconds, Status: {response.status_code}")

    except Exception as e:
        print(f"Request {i+1} with search term '{search_term}' failed due to {str(e)}")

# Calculate the average time taken per request
average_time = total_time / total_requests
success_rate = (success_count / total_requests) * 100

# Print the results
print(f"\nTotal Requests: {total_requests}")
print(f"Successful Requests: {success_count}")
print(f"Average Time per Request: {average_time:.2f} seconds")
print(f"Success Rate: {success_rate:.2f}%")