import os
import requests
import json
import re

from bs4 import BeautifulSoup
from dotenv import load_dotenv
from playwright.sync_api import sync_playwright
from typing import Callable, Any

from pif_compiler.functions.common_log import get_logger
from pif_compiler.functions.db_utils import log_ricerche

log = get_logger()
load_dotenv()

BASE_SEARCH = "https://chem.echa.europa.eu/api-substance/v1/substance?pageIndex=1&pageSize=100&searchText="
BASE_DOSSIER = "https://chem.echa.europa.eu/api-dossier-list/v1/dossier?pageIndex=1&pageSize=100&rmlId="
SUBSTANCE_SUMMARY = "https://chem.echa.europa.eu/api-substance/v1/substance/" #+id
CLASSIFICATION_ID = "https://chem.echa.europa.eu/api-cnl-inventory/prominent/overview/classifications/harmonised/459160"
TOXICOLOGICAL_INFO = "https://chem.echa.europa.eu/html-pages-prod/e4c88c6e-06c7-4daa-b0fb-1a55459ac22f/documents/IUC5-5f55d8ec-7a71-4e2c-9955-8469ead9fe84_0035f3f8-7467-4944-9028-1db2e9c99565.html" # external + rootkey
REPEATED_DOSE = "https://chem.echa.europa.eu/html-pages-prod/e4c88c6e-06c7-4daa-b0fb-1a55459ac22f/documents/IUC5-82402b09-8d8f-495c-b673-95b205be60e0_0035f3f8-7467-4944-9028-1db2e9c99565.html"

active = "&registrationStatuses=Active"
inactive = "&registrationStatuses=Inactive"
legislation = "&legislation=REACH"

#region ECHA scraping functions

def search_substance(cas : str) -> dict:
    response = requests.get(BASE_SEARCH + cas)
    if response.status_code != 200:
        log.error(f"search_substance CAS={cas}: HTTP {response.status_code}")
        return {}
    else:
        response = response.json()
        if response['state']['totalItems'] == 0:
            log.warning(f"search_substance CAS={cas}: nessuna sostanza trovata su ECHA")
            return {}
        else:
            for result in response['items']:
                if result["substanceIndex"]["rmlCas"] == cas:
                    substance = {
                        "rmlCas": result["substanceIndex"]["rmlCas"],
                        "rmlId": result["substanceIndex"]["rmlId"],
                        "rmlEc": result["substanceIndex"]["rmlEc"],
                        "rmlName": result["substanceIndex"]["rmlName"],
                        "rmlId": result["substanceIndex"]["rmlId"]
                    }
                    log.debug(f"search_substance CAS={cas}: trovata '{substance['rmlName']}'")
                    return substance
    log.warning(f"search_substance CAS={cas}: {response['state']['totalItems']} risultati ma nessun match esatto sul CAS")
    return {}


def get_dossier_info(rmlId: str, type = active) -> dict:
    url = BASE_DOSSIER + rmlId + type + legislation
    response_dossier = requests.get(url)
    if response_dossier.status_code != 200:
        log.error(f"get_dossier_info rmlId={rmlId}: HTTP {response_dossier.status_code}")
        return {}
    response_dossier_json = response_dossier.json()
    if response_dossier_json['state']['totalItems'] == 0:
        if type == active:
            log.debug(f"get_dossier_info rmlId={rmlId}: nessun dossier attivo, provo inattivi")
            return get_dossier_info(rmlId, inactive)
        log.warning(f"get_dossier_info rmlId={rmlId}: nessun dossier trovato (né attivo né inattivo)")
        return {}
    dossier_info = {}
    for dossier in response_dossier_json['items']:
        if dossier['reachDossierInfo']['dossierSubtype'] == "Article 10 - full" and dossier['reachDossierInfo']['registrationRole'] == "Lead (joint submission)":
            dossier_info = {
                "lastUpdatedDate": dossier['lastUpdatedDate'],
                "registrationStatus": dossier['registrationStatus'],
                "registrationStatusChangedDate": dossier['registrationStatusChangedDate'],
                "registrationRole": dossier['reachDossierInfo']['registrationRole'],
                "assetExternalId": dossier['assetExternalId'],
                "rootKey": dossier['rootKey']
            }
    if not dossier_info:
        log.warning(f"get_dossier_info rmlId={rmlId}: nessun dossier 'Article 10 - full / Lead' tra i {response_dossier_json['state']['totalItems']} trovati")
    return dossier_info


def get_substance_index(assetExternalId : str) -> dict:
    INDEX = "https://chem.echa.europa.eu/html-pages-prod/" + assetExternalId
    LINK_DOSSIER = INDEX + "/documents/"

    response = requests.get(INDEX + "/index.html")
    if response.status_code != 200:
        log.error(f"get_substance_index {assetExternalId}: HTTP {response.status_code}")
        return {}

    soup = BeautifulSoup(response.content, 'html.parser')
    index_data = {}

    # Toxicological information : txi
    try:
        txi_div = soup.find('div', id='id_7_Toxicologicalinformation')
        txi_link = txi_div.find('a', class_='das-leaf')
        txi_href = txi_link['href']
        index_data['toxicological_information_link'] = LINK_DOSSIER + txi_href + '.html'
    except Exception as e:
        log.warning(f"get_substance_index: link tossicologia non trovato — {e}")
        index_data['toxicological_information_link'] = None

    # Repeated dose toxicity : rdt
    try:
        rdt_div = soup.find('div', id='id_75_Repeateddosetoxicity')
        rdt_link = rdt_div.find('a', class_='das-leaf')
        rdt_href = rdt_link['href']
        index_data['repeated_dose_toxicity_link'] = LINK_DOSSIER + rdt_href + '.html'
    except Exception as e:
        log.warning(f"get_substance_index: link repeated dose non trovato — {e}")
        index_data['repeated_dose_toxicity_link'] = None

    # Acute toxicity : at
    try:
        at_div = soup.find('div', id='id_72_AcuteToxicity')
        at_link = at_div.find('a', class_='das-leaf')
        at_href = at_link['href']
        index_data['acute_toxicity_link'] = LINK_DOSSIER + at_href + '.html'
    except Exception as e:
        log.warning(f"get_substance_index: link acute toxicity non trovato — {e}")
        index_data['acute_toxicity_link'] = None

    return index_data



#endregion

#region ECHA parsing functions of html pages

def get_field_name(field_div):
    """Extract field name from the class attribute of label div"""
    label_div = field_div.find('div', class_='das-field_label')
    if not label_div:
        return None

    classes = label_div.get('class', [])

    for cls in classes:
        if cls not in ['das-field_label', 'das-empty-value', 'das-empty-label']:
            return cls

    return None


def extract_field_value(field_div):
    """Extract value from a das-field div"""
    field_name = get_field_name(field_div)
    if not field_name:
        return None

    # Skip OriginalStudy fields
    if field_name == 'OriginalStudy':
        return None

    value_div = field_div.find('div', class_='das-field_value')
    if not value_div:
        return None

    # Exclude redacted/not publishable
    redacted = value_div.find('span', class_='das-redacted-value')
    if redacted:
        return None

    # Check if empty
    empty_span = value_div.find('span', class_='das-empty-value')
    if empty_span and not value_div.find('span', class_='das-redacted-value'):
        return {field_name: ""}

    # Extract pick-list value
    pick_list = value_div.find('span', class_='das-field_value_pick-list')
    if pick_list:
        phrase = pick_list.find('span', class_='phrase')
        if phrase:
            return {field_name: phrase.get_text(strip=True)}
        if pick_list.find('span', class_='das-empty-value'):
            return {field_name: ""}

    # Extract quantity value (value + unit)
    quantity = value_div.find('span', class_='i6PhysicalQuantity')
    if quantity:
        value_span = quantity.find('span', class_='value')
        unit_span = quantity.find('span', class_='unit')

        value_text = value_span.get_text(strip=True) if value_span else ""
        unit_text = ""
        if unit_span:
            unit_phrase = unit_span.find('span', class_='phrase')
            if unit_phrase:
                unit_text = unit_phrase.get_text(strip=True)
            elif unit_span.find('span', class_='das-empty-value'):
                unit_text = ""

        if value_text:
            return {field_name: {"value": value_text, "unit": unit_text}}
        else:
            return {field_name: ""}

    # Extract checkbox value
    checkbox_checked = value_div.find('span', class_='das-value_checkbox-checked')
    checkbox_unchecked = value_div.find('span', class_='das-value_checkbox-unchecked')
    if checkbox_checked is not None or checkbox_unchecked is not None:
        return {field_name: checkbox_checked is not None}

    # Extract decimal/numeric value
    if 'das-field_decimal' in field_div.get('class', []) or 'das-field_text' in field_div.get('class', []):
        text = value_div.get_text(strip=True)
        if '[Empty]' in text or not text:
            return {field_name: ""}
        return {field_name: text}

    # Extract HTML/text content
    if value_div.find('div', class_='das-field_value_html'):
        html_content = value_div.find('div', class_='das-field_value_html')
        text = html_content.get_text(separator=' ', strip=True)
        text = re.sub(r'\[Empty\]', '', text).strip()
        if not text:
            return {field_name: ""}
        return {field_name: text}

    # Default: get text content
    text = value_div.get_text(strip=True)
    text = re.sub(r'\[Empty\]', '', text).strip()
    return {field_name: text if text else ""}


def extract_table_data(table):
    """Extract table data as array of objects"""
    rows = table.find_all('tr')
    if len(rows) < 2:
        return []

    header_row = rows[0]
    headers = []
    for th in header_row.find_all('td'):
        header_text = th.get_text(strip=True)
        headers.append(header_text)

    data = []
    for row in rows[1:]:
        cells = row.find_all('td')

        if len(cells) == 1 and cells[0].get('colspan'):
            continue

        if len(cells) == len(headers):
            row_data = {}
            for i, cell in enumerate(cells):
                cell_text = cell.get_text(strip=True)
                row_data[headers[i]] = cell_text
            data.append(row_data)

    return data


def extract_section(section):
    """Recursively extract data from a section"""
    section_data = {}

    label_h3 = section.find('h3', class_='das-block_label', recursive=False)
    if label_h3:
        section_data['label'] = label_h3.get_text(strip=True)

    direct_fields = section.find_all('div', class_='das-field', recursive=False)
    for field in direct_fields:
        field_data = extract_field_value(field)
        if field_data:
            section_data.update(field_data)

    tables = section.find_all('table', recursive=False)
    for i, table in enumerate(tables):
        table_data = extract_table_data(table)
        if table_data:
            table_key = f'table_{i+1}' if len(tables) > 1 else 'table'
            section_data[table_key] = table_data

    nested_sections = section.find_all('section', class_='das-block', recursive=False)
    if nested_sections:
        section_data['subsections'] = []
        for nested in nested_sections:
            nested_data = extract_section(nested)
            if nested_data:
                section_data['subsections'].append(nested_data)

    return section_data


def parse_toxicology_html(html_content):
    """Main function to parse the toxicological HTML document"""
    soup = BeautifulSoup(html_content, 'html.parser')

    result = {}

    title = soup.find('h4', class_='document-header')
    if title:
        result['document_title'] = title.get_text(strip=True)

    article = soup.find('article', class_='das-document')
    if not article:
        return result

    top_sections = article.find_all('section', class_='das-block', recursive=False)
    result['sections'] = []

    for section in top_sections:
        section_data = extract_section(section)
        if section_data:
            result['sections'].append(section_data)

    return result


def parse_value_with_unit(value_str: str) -> tuple:
    """Parse a combined value+unit string like '5,040mg/kg bw/day' into (value, unit)."""
    if not value_str:
        return ("", "")
    
    # Pattern to match numeric value (with commas/decimals) followed by unit
    match = re.match(r'^([\d,\.]+)\s*(.*)$', value_str.strip())
    if match:
        numeric_value = match.group(1).replace(',', '')  # Remove commas
        unit = match.group(2).strip()
        return (numeric_value, unit)
    return (value_str, "")


def extract_levels(data: dict | list, extractor: Callable[[dict, list], dict | None]) -> dict:
    """
    Generic function to recursively extract data from nested JSON structures.
    
    Args:
        data: The JSON data (dict or list) to parse
        extractor: A callable that receives (obj, path) and returns:
                   - A dict with extracted data if the object matches criteria
                   - None if no match (continue searching)
                   The 'path' is a list of labels encountered in the hierarchy.
    
    Returns:
        A dict keyed by context path (labels joined by " > "), with values
        being whatever the extractor returns.
    
    Example:
        def my_extractor(obj, path):
            if obj.get("SomeField"):
                return {"field": obj["SomeField"]}
            return None
        
        results = extract_from_json(data, my_extractor)
    """
    results = {}
    
    def recurse(obj: Any, path: list = None):
        if path is None:
            path = []
            
        if isinstance(obj, dict):
            # Check for label to use in path
            current_label = obj.get("label", "")
            current_path = path + [current_label] if current_label else path
            
            # Call the extractor function
            extracted = extractor(obj, current_path)
            if extracted is not None:
                key = " > ".join(filter(None, current_path)) or "root"
                results[key] = extracted
            
            # Recurse into all values
            for k, val in obj.items():
                if k != "label":
                    recurse(val, current_path)
                
        elif isinstance(obj, list):
            for item in obj:
                recurse(item, path)
    
    recurse(data)
    return results


def rdt_extractor(obj: dict, path: list) -> dict | None:
    indicator = obj.get("EffectLevelUnit", "").strip()
    value_str = obj.get("EffectLevelValue", "").strip()
    
    if indicator and value_str:
        numeric_value, unit = parse_value_with_unit(value_str)
        
        # Extract route (last label) and toxicity_type (second-to-last label)
        filtered_path = [p for p in path if p]
        route = filtered_path[-1] if len(filtered_path) >= 1 else ""
        toxicity_type = filtered_path[-2] if len(filtered_path) >= 2 else ""
        
        return {
            "indicator": indicator,
            "value": numeric_value,
            "unit": unit,
            "route": route,
            "toxicity_type": toxicity_type
        }
    return None

def at_extractor(obj: dict, path: list) -> dict | None:
    indicator = obj.get("EffectLevelUnit", "").strip()
    value_str = obj.get("EffectLevelValue", "").strip()
    
    if indicator and value_str:
        numeric_value, unit = parse_value_with_unit(value_str)
        
        filtered_path = [p for p in path if p]
        route = filtered_path[-1] if len(filtered_path) >= 1 else ""
        route = route.replace("Acute toxicity: via ", "").strip()

        return {
            "indicator": indicator,
            "value": numeric_value,
            "unit": unit,
            "route": route
        }
    return None

#endregion

#region Orchestrator functions

def echa_flow(cas) -> dict:
    try:
        substance = search_substance(cas)
        dossier_info = get_dossier_info(substance['rmlId'])
        index = get_substance_index(dossier_info['assetExternalId'])
    except KeyError as e:
        log.error(f"echa_flow CAS={cas}: chiave mancante nella risposta ECHA — {e}")
        return {}

    result = {
        "substance": substance,
        "dossier_info": dossier_info,
        "index": index,
        "toxicological_information": {},
        "acute_toxicity": {},
        "repeated_dose_toxicity": {}
    }

    # Fetch and parse toxicological information
    txi_link = index.get('toxicological_information_link')
    if txi_link:
        response_summary = requests.get(txi_link)
        if response_summary.status_code == 200:
            result['toxicological_information'] = parse_toxicology_html(response_summary.content)
        else:
            log.warning(f"echa_flow CAS={cas}: tossicologia HTTP {response_summary.status_code}")

    # Fetch and parse acute toxicity
    at_link = index.get('acute_toxicity_link')
    if at_link:
        response_acute = requests.get(at_link)
        if response_acute.status_code == 200:
            result['acute_toxicity'] = parse_toxicology_html(response_acute.content)
        else:
            log.warning(f"echa_flow CAS={cas}: acute toxicity HTTP {response_acute.status_code}")

    # Fetch and parse repeated dose toxicity
    rdt_link = index.get('repeated_dose_toxicity_link')
    if rdt_link:
        response_repeated = requests.get(rdt_link)
        if response_repeated.status_code == 200:
            result['repeated_dose_toxicity'] = parse_toxicology_html(response_repeated.content)
        else:
            log.warning(f"echa_flow CAS={cas}: repeated dose HTTP {response_repeated.status_code}")

    txi_ok = bool(result['toxicological_information'])
    at_ok = bool(result['acute_toxicity'])
    rdt_ok = bool(result['repeated_dose_toxicity'])
    log.info(f"echa_flow CAS={cas}: txi={'OK' if txi_ok else '-'}, acute={'OK' if at_ok else '-'}, rdt={'OK' if rdt_ok else '-'}")

    return result

def cas_validation(cas: str) -> str:
    if cas is None or cas.strip() == "":
        log.error("cas_validation: CAS vuoto o None")
        return None
    cas_stripped = cas.replace("-", "")
    if cas_stripped.isdigit() and len(cas_stripped) <= 12:
        return cas.strip()
    log.error(f"cas_validation: CAS '{cas}' non valido (formato non riconosciuto)")
    return None

def orchestrator(cas: str) -> dict:
    log.debug(f"ECHA orchestrator CAS={cas}")
    cas_validated = cas_validation(cas)
    if not cas_validated:
        return None

    echa_data = echa_flow(cas_validated)
    if echa_data:
        log.info(f"ECHA CAS={cas}: completato")
        log_ricerche(cas, 'ECHA', True)
        return echa_data
    else:
        log.error(f"ECHA CAS={cas}: nessun dato recuperato")
        log_ricerche(cas, 'ECHA', False)
        return None
#endregion

if __name__ == "__main__":
    cas_test = "113170-55-1"
    result = orchestrator(cas_test)