debug_html_structure.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Debug HTML Structure for PHIN Search Results

This script will examine the actual HTML structure to understand
how consultant data is organized and fix the parsing issues.
"""

import requests
from bs4 import BeautifulSoup
import json
import re

BASE = "https://www.phin.org.uk"
SEARCH_URL = BASE + "/search/consultants"

UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
)

HEADERS = {
    "User-Agent": UA,
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-GB,en;q=0.9",
    "Cache-Control": "no-cache",
    "Pragma": "no-cache",
    "Connection": "keep-alive",
}

def get_search_page():
    """Get the search page HTML."""
    url = "https://www.phin.org.uk/search/consultants?f_distance=360&s_location_input=United%20Kingdom&s_location_coordinates=55.378051%2C-3.435973&s_procedure_input=Breast%20implants&s_procedure_id=7&s_page_number=1"
    resp = requests.get(url, headers=HEADERS)
    return resp.text if resp.status_code == 200 else None

def debug_html_structure():
    """Debug the HTML structure to understand the data layout."""
    
    html_content = get_search_page()
    if not html_content:
        print("Failed to get HTML content")
        return
    
    soup = BeautifulSoup(html_content, 'html.parser')
    
    print("=== HTML Structure Debug ===")
    print(f"HTML length: {len(html_content)} characters")
    
    # Look for the total count
    print(f"\n=== Looking for Total Count ===")
    count_patterns = [
        r'(\d+)\s+results?',
        r'Showing\s+(\d+)',
        r'Found\s+(\d+)',
        r'(\d+)\s+consultants?',
        r'Load more\s+(\d+)',
        r'(\d+)\s*\.\.\.\s*(\d+)',  # Pagination like "12345 ... 41"
    ]
    
    for pattern in count_patterns:
        matches = re.findall(pattern, html_content, re.IGNORECASE)
        if matches:
            print(f"Pattern '{pattern}': {matches}")
    
    # Look for consultant names in the HTML
    print(f"\n=== Looking for Consultant Names ===")
    name_patterns = [
        r'Mr\s+[A-Z][a-z]+\s+[A-Z][a-z]+',
        r'Ms\s+[A-Z][a-z]+\s+[A-Z][a-z]+',
        r'Dr\s+[A-Z][a-z]+\s+[A-Z][a-z]+',
    ]
    
    for pattern in name_patterns:
        matches = re.findall(pattern, html_content)
        if matches:
            print(f"Found names with pattern '{pattern}': {matches[:10]}")  # Show first 10
    
    # Look for consultant IDs in URLs
    print(f"\n=== Looking for Consultant IDs in URLs ===")
    id_patterns = [
        r'/profiles/consultants/[^/]+-(\d+)',
        r'/profiles/consultants/(\d+)',
        r'consultant[_-]?id[=:](\d+)',
        r'id[=:](\d+)',
    ]
    
    for pattern in id_patterns:
        matches = re.findall(pattern, html_content)
        if matches:
            print(f"Found IDs with pattern '{pattern}': {matches[:10]}")  # Show first 10
    
    # Examine search result elements
    print(f"\n=== Examining Search Result Elements ===")
    search_results = soup.find_all('div', class_='search-result')
    print(f"Found {len(search_results)} search-result elements")
    
    for i, result in enumerate(search_results[:3]):  # Examine first 3
        print(f"\n--- Search Result {i+1} ---")
        print(f"Classes: {result.get('class', [])}")
        
        # Look for all links
        links = result.find_all('a', href=True)
        print(f"Links found: {len(links)}")
        for j, link in enumerate(links[:5]):  # Show first 5 links
            href = link.get('href', '')
            text = link.get_text(strip=True)
            print(f"  Link {j+1}: {href} (text: '{text}')")
        
        # Look for all text content
        text_content = result.get_text()
        print(f"Text content length: {len(text_content)}")
        print(f"Text preview: {text_content[:200]}...")
        
        # Look for specific data attributes
        data_attrs = [attr for attr in result.attrs.keys() if attr.startswith('data-')]
        if data_attrs:
            print(f"Data attributes: {data_attrs}")
    
    # Look for pagination information
    print(f"\n=== Pagination Information ===")
    pagination_elements = soup.find_all(['div', 'nav', 'span'], class_=re.compile(r'pagination|page|load'))
    for elem in pagination_elements:
        print(f"Pagination element: {elem.name} - {elem.get_text().strip()}")
    
    # Look for any JavaScript data
    print(f"\n=== JavaScript Data ===")
    script_tags = soup.find_all('script')
    for script in script_tags:
        if script.string and ('consultant' in script.string.lower() or 'procedure' in script.string.lower()):
            print(f"Script with potential data:")
            print(f"  Content preview: {script.string[:300]}...")
    
    # Save a sample of the HTML for manual inspection
    with open('debug_html_sample.html', 'w', encoding='utf-8') as f:
        f.write(html_content)
    print(f"\nHTML saved to: debug_html_sample.html")

def extract_consultant_data_correctly():
    """Extract consultant data using the correct HTML structure."""
    
    html_content = get_search_page()
    if not html_content:
        return []
    
    soup = BeautifulSoup(html_content, 'html.parser')
    consultants = []
    
    # Find all search result elements
    search_results = soup.find_all('div', class_='search-result')
    
    print(f"\n=== Extracting Consultant Data ===")
    print(f"Found {len(search_results)} search result elements")
    
    for i, result in enumerate(search_results):
        consultant_data = {
            "result_index": i + 1,
            "extracted_from": "corrected_parsing"
        }
        
        # Get all links in this result
        links = result.find_all('a', href=True)
        
        # Look for consultant profile links
        for link in links:
            href = link.get('href', '')
            if '/profiles/consultants/' in href:
                # Extract consultant ID from the URL
                # URLs like: /profiles/consultants/neil-cahoon-12345?procedureId=7
                id_match = re.search(r'/profiles/consultants/[^/]+-(\d+)', href)
                if id_match:
                    consultant_id = int(id_match.group(1))
                    consultant_data['consultant_id'] = consultant_id
                    consultant_data['profile_url'] = href
                    break
        
        # Extract consultant name from the link text or nearby elements
        for link in links:
            if '/profiles/consultants/' in link.get('href', ''):
                name_text = link.get_text(strip=True)
                if name_text and len(name_text) > 3:  # Avoid empty or very short names
                    consultant_data['name'] = name_text
                    break
        
        # Extract other data from the result text
        result_text = result.get_text()
        
        # Extract specialty
        if 'Plastic surgery' in result_text:
            consultant_data['specialty'] = 'Plastic surgery'
        elif 'General surgery' in result_text:
            consultant_data['specialty'] = 'General surgery'
        
        # Extract admissions
        admission_match = re.search(r'Admissions[:\s]*(\d+|7 or fewer)', result_text, re.IGNORECASE)
        if admission_match:
            admissions_text = admission_match.group(1)
            if admissions_text == "7 or fewer":
                consultant_data['admissions'] = "7 or fewer"
            else:
                consultant_data['admissions'] = int(admissions_text)
        
        # Extract satisfaction percentage
        satisfaction_match = re.search(r'(\d+)%', result_text)
        if satisfaction_match:
            consultant_data['satisfaction_percentage'] = int(satisfaction_match.group(1))
        
        # Extract distance
        distance_match = re.search(r'(\d+)\s*miles?', result_text, re.IGNORECASE)
        if distance_match:
            consultant_data['distance_miles'] = int(distance_match.group(1))
        
        # Extract consultation fees
        fee_match = re.search(r'£(\d+(?:-\d+)?)', result_text)
        if fee_match:
            consultant_data['consultation_fee'] = fee_match.group(1)
        elif 'No charge' in result_text:
            consultant_data['consultation_fee'] = "No charge"
        
        # Extract remote consultation availability
        if 'remote' in result_text.lower():
            consultant_data['remote_consultations'] = True
        
        # Only add if we have a consultant ID
        if consultant_data.get('consultant_id'):
            consultants.append(consultant_data)
    
    print(f"Extracted {len(consultants)} consultants with valid IDs")
    
    # Show sample data
    if consultants:
        print(f"\nSample consultant data:")
        for i, consultant in enumerate(consultants[:5]):
            print(f"  {i+1}. {consultant}")
    
    return consultants

def main():
    print("PHIN HTML Structure Debug")
    print("=" * 40)
    
    # Debug the HTML structure
    debug_html_structure()
    
    # Extract consultant data correctly
    consultants = extract_consultant_data_correctly()
    
    # Save results
    if consultants:
        with open('corrected_consultant_data.json', 'w', encoding='utf-8') as f:
            json.dump(consultants, f, indent=2, ensure_ascii=False)
        print(f"\nCorrected consultant data saved to: corrected_consultant_data.json")

if __name__ == "__main__":
    main()