Web Scraping at Scale with Python: Processing 500+ Records Daily Without Getting Blocked

import asyncio
import aiohttp
from urllib.robotparser import RobotFileParser
from datetime import datetime, timedelta

class RespectfulScraper:
    def __init__(self, base_url: str, requests_per_minute: int = 10):
        self.base_url = base_url
        self.rpm = requests_per_minute
        self.last_request_time = None
        self.min_interval = 60.0 / requests_per_minute

        # Parse robots.txt
        self.robot_parser = RobotFileParser()
        self.robot_parser.set_url(f"{base_url}/robots.txt")
        self.robot_parser.read()

    def can_fetch(self, url: str) -> bool:
        """Check if we're allowed to scrape this URL"""
        return self.robot_parser.can_fetch("SalesBridgeBot/1.0", url)

    async def fetch(self, session: aiohttp.ClientSession, url: str) -> str:
        """Fetch a URL with rate limiting and polite headers"""
        if not self.can_fetch(url):
            raise PermissionError(f"robots.txt disallows: {url}")

        # Enforce rate limit
        if self.last_request_time:
            elapsed = (datetime.now() - self.last_request_time).total_seconds()
            if elapsed < self.min_interval:
                await asyncio.sleep(self.min_interval - elapsed)

        headers = {
            'User-Agent': 'SalesBridgeBot/1.0 (+https://salesbridge.ai/bot)',
            'Accept': 'text/html,application/xhtml+xml',
            'Accept-Language': 'en-US,en;q=0.9',
        }

        self.last_request_time = datetime.now()

        async with session.get(url, headers=headers) as response:
            if response.status == 429:  # Too Many Requests
                retry_after = int(response.headers.get('Retry-After', 60))
                await asyncio.sleep(retry_after)
                return await self.fetch(session, url)  # Retry

            response.raise_for_status()
            return await response.text()

from playwright.async_api import async_playwright

class DynamicScraper:
    async def scrape_js_rendered(self, url: str, wait_for: str = None):
        """Scrape content that requires JavaScript rendering"""
        async with async_playwright() as p:
            browser = await p.chromium.launch(
                headless=True,
                args=[
                    '--disable-gpu',
                    '--no-sandbox',
                    '--disable-dev-shm-usage',
                ]
            )

            context = await browser.new_context(
                viewport={'width': 1280, 'height': 720},
                user_agent='Mozilla/5.0 (compatible; SalesBridgeBot/1.0)',
            )

            page = await context.new_page()

            # Block unnecessary resources to speed up scraping
            await page.route("**/*.{png,jpg,jpeg,gif,svg,css,font,woff,woff2}",
                           lambda route: route.abort())

            await page.goto(url, wait_until='networkidle')

            if wait_for:
                await page.wait_for_selector(wait_for, timeout=10000)

            content = await page.content()
            await browser.close()

            return content

from pydantic import BaseModel, validator, HttpUrl
from datetime import date
from typing import Optional, List

class Opportunity(BaseModel):
    title: str
    agency: str
    solicitation_number: str
    response_deadline: date
    url: HttpUrl
    naics_codes: List[str] = []
    estimated_value: Optional[float] = None

    @validator('title')
    def title_not_empty(cls, v):
        if len(v.strip()) < 10:
            raise ValueError('Title too short - likely parsing error')
        return v.strip()

    @validator('response_deadline')
    def deadline_in_future(cls, v):
        if v < date.today():
            raise ValueError('Deadline has already passed')
        return v

    @validator('solicitation_number')
    def valid_solicitation(cls, v):
        # Government solicitation numbers follow specific patterns
        if not any(v.startswith(prefix) for prefix in ['W', 'FA', 'N', 'SP', 'HQ']):
            raise ValueError(f'Unusual solicitation format: {v}')
        return v

class DeduplicationEngine:
    def __init__(self, db_connection):
        self.db = db_connection

    async def is_duplicate(self, opportunity: Opportunity) -> bool:
        """Check if we've already seen this opportunity"""
        existing = await self.db.find_one({
            'solicitation_number': opportunity.solicitation_number
        })
        return existing is not None

    async def upsert(self, opportunity: Opportunity):
        """Insert or update opportunity"""
        await self.db.update_one(
            {'solicitation_number': opportunity.solicitation_number},
            {'$set': opportunity.dict(), '$setOnInsert': {'first_seen': date.today()}},
            upsert=True
        )