import fnmatch
import hashlib
import json
import logging
import os
import re
import subprocess
import sys
import time
import uuid
from functools import lru_cache
from pathlib import Path
from typing import Any, Dict, List, Optional

from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.concurrency import run_in_threadpool
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
import asyncio

from sqlalchemy.orm.attributes import flag_modified
from app.api.config_analyzer import detect_config_type, get_rules_for_config
from app.auth import get_current_user
from app.models import User
from app.services.license_limits import enforce_scan_limits
from app.services.license_validator import LicenseValidator
from app.users import auth_backend, fastapi_users, get_user_manager
from app.code_analyzer.code_analysis_settings import code_analysis_settings
from app.database import SessionDep
from app.models import PatchType, ScanRecord, ScanType
from app.utils.patch_db_utils import save_patch_to_db
from app.utils.path_security import validate_and_normalize_path, sanitize_error_message
from app.utils.code_analysis_utils import extract_fix_records
from app.utils.comment_patch_generator import generate_comment_patch
from app.utils.patch_utils import findings_to_patch
from app.utils.input_validation import (
    sanitize_extensions,
    sanitize_excluded_dirs,
    validate_config,
)
from app.utils.rate_limiter import scan_rate_limiter, patch_rate_limiter
from app.prompts.unified_prompt import build_unified_fix_prompt
from app.services.llm_service import generate_completion, LLMConfig
from app.code_analyzer.custom import (
    SQLCryptoScanner, HTMLCryptoScanner, ApexCryptoScanner,
    ElixirCryptoScanner, CertificateCryptoScanner,
    BouncyCastleCryptoScanner, ParamikoCryptoScanner,
)
from app.code_analyzer.engine import _opengrep_exe

logger = logging.getLogger(__name__)

router = APIRouter()

# ─────────────────────────────────────────────────────────────────────
# Active scan cancellation registry
# ─────────────────────────────────────────────────────────────────────
# Maps scan_id -> cancelled flag. Checked by the streaming scan loop.
_cancelled_scans: Dict[str, bool] = {}


def _mark_scan_cancelled(scan_id: str) -> None:
    """Mark a scan as cancelled so the streaming loop stops."""
    _cancelled_scans[scan_id] = True


def _is_scan_cancelled(scan_id: str) -> bool:
    """Check if a scan has been cancelled."""
    return _cancelled_scans.get(scan_id, False)


def _cleanup_scan(scan_id: str) -> None:
    """Remove a scan from the cancellation registry."""
    _cancelled_scans.pop(scan_id, None)


@router.post("/code/scan/cancel/{scan_id}")
async def cancel_scan(scan_id: str, session: SessionDep = None):
    """Cancel a running code analysis scan.

    The streaming scan loop checks this flag between batches
    and saves partial results before stopping.

    For very fast scans that already completed before this request
    arrives, we retroactively mark the saved record as partial.
    """
    _mark_scan_cancelled(scan_id)
    logger.info(f"Scan {scan_id} marked for cancellation")

    # Retroactively mark already-saved scan as partial.
    # For tiny scans (2-5 files) the scan may finish and save as
    # "complete" before the user can click cancel.  Fix it here.
    retroactive = False
    if session:
        try:
            from sqlmodel import select
            stmt = select(ScanRecord).where(
                ScanRecord.scan_type == ScanType.CODE_ANALYSIS
            ).order_by(ScanRecord.timestamp.desc()).limit(50)  # type: ignore[arg-type]
            records = session.exec(stmt).all()
            for record in records:
                sd = record.scan_data or {}
                if sd.get("scan_id") == scan_id and not sd.get("is_partial"):
                    sd["is_partial"] = True
                    sd["partial_reason"] = "cancelled"
                    record.scan_data = sd
                    flag_modified(record, "scan_data")
                    session.add(record)
                    session.commit()
                    session.refresh(record)
                    retroactive = True
                    logger.info(f"Retroactively marked scan {scan_id} (record {record.id}) as partial")
                    break
        except Exception as e:
            logger.warning(f"Failed to retroactively mark scan as partial: {e}")

    return {"success": True, "message": "Cancellation requested", "retroactive": retroactive}

# --- Input Sanitization (moved to utils/input_validation.py) ---


# Word-boundary regex for PQC/crypto classification (mirrors code_analysis_utils._PQC_PATTERN)
# Intentionally broad: better to catch a few general issues than miss crypto findings
_PQC_PATTERN = re.compile(
    r"\b("
    r"quantum|post-quantum|shor|grover"
    r"|crypto(?:graphy|graphic(?:ally)?)?"
    r"|cipher(?:text)?|decipher|aes|rsa|hash(?:ing)?|md5|sha[\d-]|des(?:ede)?"
    r"|key.?size|encrypt(?:ion|ed)?|decrypt"
    r"|tls|ssl|hmac|pbkdf|diffie.?hellman"
    r"|ecdsa|ecdh|dsa|blowfish|3des|triple.?des"
    r"|private.?key|public.?key|priv.?key|certificates?"
    r"|gcm|cbc|ecb|ctr"
    r"|chacha|poly1305|bcrypt|scrypt|argon2?"
    r"|x509|pkcs|nonce"
    r")\b",
    re.IGNORECASE,
)


_PQC_RULE_PATH_KEYWORDS = re.compile(
    r"(crypto|cipher|hash|quantum|tls|ssl|certificate|jwt|jsonwebtoken|key-size|encrypt)",
    re.IGNORECASE,
)


def _is_pqc_rule_path(rule_path: str) -> bool:
    """Check if a rule path is PQC/crypto-related based on path keywords.

    Only inspects the rule-relative portion of the path (after custom-rules/
    or rules/) to avoid false positives from project directory names
    like 'QUANTUMCOMPUTING'.
    """
    # Registry paths (p/*, r/*) — match as-is
    if rule_path.startswith(('p/', 'r/', 'http://', 'https://')):
        return bool(_PQC_RULE_PATH_KEYWORDS.search(rule_path))
    # Local paths — strip base directory prefix, keep only rule-relative part
    normalized = rule_path.replace('\\', '/')
    for marker in ('custom-rules/', 'rules/'):
        idx = normalized.find(marker)
        if idx >= 0:
            normalized = normalized[idx:]
            break
    return bool(_PQC_RULE_PATH_KEYWORDS.search(normalized))


def _classify_finding_category(finding: Dict[str, Any]) -> str:
    """Classify a finding as 'pqc' or 'general' using rule name and metadata."""
    metadata = finding.get("extra", {}).get("metadata", {})
    message = finding.get("extra", {}).get("message", "") or finding.get("message", "")
    raw_check_id = finding.get("check_id", "")
    rule_name = raw_check_id.rsplit(".", 1)[-1] if raw_check_id else ""

    if metadata.get("quantum_vulnerable") or _PQC_PATTERN.search(message) or _PQC_PATTERN.search(rule_name):
        return "pqc"
    return "general"



def enrich_findings(
    findings: List[Dict[str, Any]],
    session: Optional[SessionDep] = None,
    user_id: Optional[str] = None,
) -> List[Dict[str, Any]]:
    """Enrich findings with metadata (has_fix flag based on extra.fix only).

    Args:
        findings: Raw findings from security scan
        session: Optional database session (not used - kept for compatibility)
        user_id: Optional user ID (not used - kept for compatibility)

    Returns:
        List of enriched findings with has_fix flag and normalized line numbers
    """
    enriched = []
    for idx, finding in enumerate(findings):
        enriched_finding = finding.copy()
        # Only mark has_fix if the scanner provided an auto-fix in extra.fix
        enriched_finding["has_fix"] = bool(finding.get("extra", {}).get("fix"))

        # CRITICAL FIX: Normalize line numbers to direct properties
        # Scanner provides start.line (nested) and end.line (nested)
        # Frontend needs direct properties to work correctly
        # This ensures line numbers survive database storage and JavaScript object access
        
        # Get line numbers from nested structure (start.line, end.line)
        start_line = None
        end_line = None
        
        if "start" in finding and isinstance(finding["start"], dict):
            if "line" in finding["start"]:
                start_line = finding["start"]["line"]
                enriched_finding["start_line"] = start_line
                
        if "end" in finding and isinstance(finding["end"], dict):
            if "line" in finding["end"]:
                end_line = finding["end"]["line"]
                enriched_finding["end_line"] = end_line

        # DEBUG: Log first few findings to see what's being normalized
        if idx < 3:
            logger.debug(f"[enrich_findings] Finding {idx}: check_id={finding.get('check_id', '?')}, start_line={start_line}, end_line={end_line}, has_start_dict={isinstance(finding.get('start'), dict)}")

        enriched.append(enriched_finding)
    
    return enriched


def create_scan_data(
    findings: List[Dict[str, Any]],
    scan_results: Dict[str, Any],
    session: Optional[SessionDep] = None,
    user_id: Optional[str] = None,
    **extra_fields
) -> Dict[str, Any]:
    """Create standardized scan data structure

    Args:
        findings: Raw scanner findings
        scan_results: Full scan results with metadata
        session: Optional database session for persisting patches
        user_id: Optional user ID for patch ownership
        **extra_fields: Additional fields to include (e.g., parent_directory_scan)

    Returns:
        Standardized scan data dict with enriched_findings
    """
    enriched = enrich_findings(findings, session=session, user_id=user_id)
    return {
        "enriched_findings": enriched,
        "findings_count": len(enriched),
        "fixable_count": len([f for f in enriched if f.get("has_fix")]),
        "errors": scan_results.get("errors", []),
        **extra_fields
    }


def save_code_scan_to_db(
    session, target_path: str, scan_data: Dict[str, Any], user_id: str, project_path: str = None
) -> str:
    """Save a security scan result to the database

    Args:
        session: Database session
        target_path: Path to the scanned file/directory
        scan_data: Scan results data
        user_id: User who initiated the scan (from JWT token)
        project_path: Project root path (for grouping scans)
    """
    scan_record = ScanRecord(
        target=target_path,
        project_path=project_path,
        scan_type=ScanType.CODE_ANALYSIS,
        scan_data=scan_data,
        user_id=user_id,
    )
    session.add(scan_record)
    session.commit()
    session.refresh(scan_record)
    return scan_record.id


def save_directory_scan_results(
    session,
    scan_results: Dict[str, Any],
    parent_scan_id: str,
    directory_path: str,
    user_id: str,
    is_partial: bool = False,
    partial_reason: Optional[str] = None,
) -> Dict[str, Any]:
    """Save directory scan results as a single directory scan record with grouped findings.

    IMPORTANT: This saves ONE record for the entire directory scan (not individual file records).
    This prevents conflicts with individual file scans on the same files.
    
    OPTIMIZATION: Clean files are stored in a separate table (scan_files) to reduce
    the size of the main scan_data JSON field and improve performance.

    Args:
    session: Database session
    scan_results: Full security scan results from directory scan
    parent_scan_id: The original scan ID from the directory scan
    directory_path: The directory that was scanned
    user_id: User who initiated the scan
    is_partial: Whether this is from a partial/cancelled scan
    partial_reason: Why the scan is partial — "cancelled" or "error"

    Returns:
        Dict with success status and statistics
    """
    # Get list of all scanned files
    scanned_files = scan_results.get("paths", {}).get("scanned", [])
    if not scanned_files and not is_partial:
        logger.warning(f"No scanned files found in directory scan {parent_scan_id}")
        return {
            "success": False,
            "reason": "No files scanned",
            "saved_count": 0,
            "total_files": 0,
            "errors": [],
        }

    # Get enriched findings
    enriched_findings = scan_results.get("enriched_findings", [])

    # CRITICAL FIX: Ensure EVERY finding has normalized line numbers before saving
    # This guarantees line numbers survive database storage for directory scans
    for finding in enriched_findings:
        # Normalize start_line if not already present
        if "start_line" not in finding and "start" in finding and isinstance(finding["start"], dict):
            if "line" in finding["start"]:
                finding["start_line"] = finding["start"]["line"]
        # Normalize end_line if not already present
        if "end_line" not in finding and "end" in finding and isinstance(finding["end"], dict):
            if "line" in finding["end"]:
                finding["end_line"] = finding["end"]["line"]

    # DEBUG: Log sample finding structure after normalization
    if enriched_findings:
        sample = enriched_findings[0]
        logger.debug(f"Directory scan: Sample finding has start_line={sample.get('start_line')}, end_line={sample.get('end_line')}")

    # OPTIMIZATION: Separate files with findings from clean files
    # Normalize paths to absolute form to handle mixed separators on Windows
    # (glob returns 'C:/path\\.envrc', scanner returns 'C:\\path\\.envrc')
    # Use directory_path as base for relative paths — os.path.abspath() resolves
    # relative to the backend CWD (inside the install dir) which is wrong.
    def to_abs(p: str) -> str:
        return os.path.normcase(p if os.path.isabs(p) else os.path.join(directory_path, p))

    files_with_findings = set()
    for finding in enriched_findings:
        file_path = finding.get("path")
        if file_path:
            files_with_findings.add(to_abs(file_path))

    clean_files = [f for f in scanned_files if to_abs(f) not in files_with_findings]
    
    logger.info(f"Directory scan optimization: {len(files_with_findings)} files with findings, {len(clean_files)} clean files")

    # Create ONE directory scan record with all findings grouped by file
    # This is the key fix - save as a single record, not per-file records
    try:
        # Store all scanned file paths so the frontend can show green dots for clean files
        all_scanned_abs = [
            f if os.path.isabs(f) else os.path.join(directory_path, f)
            for f in scanned_files
        ]
        directory_scan_data = {
            "success": True,
            "data": {
                "paths": {"scanned": all_scanned_abs},  # All scanned files (findings + clean)
                "version": scan_results.get("version"),
                "engine_kind": scan_results.get("engine_kind"),
            },
            "language_detected": None,  # Multiple languages in directory scan
            "rules_used": [],  # Auto-detected for directory scans
            "scan_id": parent_scan_id,
            "findings_count": len(enriched_findings),
            "fixable_count": len([f for f in enriched_findings if f.get("has_fix")]),
            "enriched_findings": enriched_findings,
            "scan_type": "directory",  # Mark as directory scan type
            "is_directory": True,
            "is_partial": is_partial,  # Mark if this is from a partial/cancelled scan
            "partial_reason": partial_reason if is_partial else None,  # "cancelled" or "error"
            "files_scanned": len(scanned_files),  # Total count
            "clean_files_count": len(clean_files),  # Count of clean files
        }

        # Save as a single directory scan record
        # Use directory_path as target so it's clearly a directory scan
        saved_scan_id = save_code_scan_to_db(session, directory_path, directory_scan_data, user_id=user_id, project_path=directory_path)
        
        # OPTIMIZATION: Save clean files to separate table
        from app.models import ScanFile
        for file_path in clean_files:
            absolute_path = file_path if os.path.isabs(file_path) else os.path.join(directory_path, file_path)
            scan_file = ScanFile(
                scan_id=saved_scan_id,
                file_path=absolute_path,
                has_findings=False
            )
            session.add(scan_file)
        
        # Also save files with findings to the table for consistency
        for file_path in files_with_findings:
            scan_file = ScanFile(
                scan_id=saved_scan_id,
                file_path=file_path,
                has_findings=True
            )
            session.add(scan_file)
        
        session.commit()
        
        logger.info(
            f"Saved directory scan as NEW record with ID={saved_scan_id}: {directory_path} ({len(enriched_findings)} findings across {len(scanned_files)} files, {len(clean_files)} clean files stored separately)"
        )

        return {
            "success": True,
            "saved_count": 1,  # One directory record
            "total_files": len(scanned_files),
            "errors": [],
        }

    except Exception as e:
        error_msg = f"Failed to save directory scan: {str(e)}"
        logger.error(error_msg)
        return {
            "success": False,
            "saved_count": 0,
            "total_files": len(scanned_files),
            "errors": [error_msg],
        }


@router.on_event("startup")
async def preload_scanner_config():
    """Preload scanner configuration at startup for faster first requests"""
    try:
        config = load_scanner_config()
        logger.info("opengrep configuration preloaded successfully at startup")
    except Exception as e:
        logger.error(f"Failed to preload opengrep configuration at startup: {str(e)}")


# Security: Path validation implemented via validate_and_normalize_path()
# Additional improvements to consider:
# - Add rate limiting to prevent abuse
# - Consider using async subprocess for better concurrency


class ScannerConfig:
    """Configuration class for analysis rules and language mappings

    This class manages the mapping between file extensions and programming languages,
    as well as the security rules available for each language.

    Attributes:
        extension_to_language_map: Maps file extensions (e.g., '.py') to languages (e.g., 'python')
        language_to_rules_map: Maps languages to their available security rule paths
    """

    def __init__(
        self,
        extension_to_language_map: Dict[str, str],
        language_to_rules_map: Dict[str, List[str]],
        timeout_seconds: int = 120,
    ):
        self.extension_to_language_map = extension_to_language_map
        self.language_to_rules_map = language_to_rules_map
        self.timeout_seconds = timeout_seconds

    def detect_language_from_extension(self, file_path: str) -> Optional[str]:
        """Detect programming language from file extension or filename.

        Handles:
          - Normal extensions: app.py -> '.py' -> python
          - Pure dotfiles: .envrc -> '.envrc' -> dotenv
          - Bare filenames: Dockerfile -> 'dockerfile' -> dockerfile
        """
        p = Path(file_path)
        file_ext = p.suffix.lower()
        if file_ext:
            return self.extension_to_language_map.get(file_ext)
        # No extension — try dotfile name first, then bare filename
        name = p.name.lower()
        if name.startswith('.') and '.' not in name[1:]:
            return self.extension_to_language_map.get(name)
        return self.extension_to_language_map.get(name)

    def get_security_rules_for_language(self, language: str) -> List[str]:
        """Get available security rules for a programming language

        Args:
            language: The programming language name

        Returns:
            List of file paths to security rule configurations

        Example:
            >>> config.get_security_rules_for_language("python")
            ['/path/to/python/lang/security', '/path/to/python/flask/security']
        """
        return self.language_to_rules_map.get(language, [])


@lru_cache()
def load_scanner_config() -> ScannerConfig:
    """Load and validate analysis rules configuration using Pydantic settings

    Returns:
        ScannerConfig: Configured analysis rules and language mappings

    Raises:
        HTTPException: If configuration loading fails
    """
    logger.info("Loading analysis rules configuration...")

    try:
        # Build extension to language mapping
        # Includes both dotted extensions (.py) and bare filenames (Dockerfile)
        extension_to_language_map: Dict[str, str] = {}
        for language, lang_config in code_analysis_settings.languages.items():
            for extension in lang_config.extensions:
                extension_to_language_map[extension.lower()] = language

        # Build language to rules mapping (filter out non-existent paths)
        language_to_rules_map: Dict[str, List[str]] = {}
        total_rules = 0
        missing_rules = []

        # Registry fallbacks when local rules aren't available (e.g. packaged builds)
        registry_fallbacks = {
            "python": ["p/python"],
            "javascript": ["p/javascript"],
            "typescript": ["p/typescript"],
            "java": ["p/java"],
            "go": ["p/golang"],
            "php": ["p/php"],
            "ruby": ["p/ruby"],
            "csharp": ["p/csharp"],
            "c": ["p/c"],
            "scala": ["p/scala"],
            "rust": ["p/rust"],
            "swift": ["p/swift"],
            "kotlin": ["p/kotlin"],
            "terraform": ["p/terraform"],
            "dockerfile": ["p/dockerfile"],
        }

        # Build list of base paths to search for rules.
        # In frozen (PyInstaller) builds the CWD is the exe directory, so relative
        # paths from code_analysis_settings are useless.  Use absolute _MEIPASS paths only.
        if getattr(sys, 'frozen', False):
            internal_dir = sys._MEIPASS
            base_paths = [
                os.path.join(internal_dir, 'rules', 'custom-rules'),
            ]
            logger.info(f"Frozen build: using absolute rule paths from {internal_dir}")
        else:
            base_paths = list(code_analysis_settings.rules_base_paths)

        for language, lang_config in code_analysis_settings.languages.items():
            available_rules = []
            for rule_path in lang_config.security_rules:
                # Check if it's a registry path (e.g., p/python, p/rust) or remote URL.
                # Local rule paths always end with .yaml — never treat those as registry
                # rules even if they start with "r/" (e.g. the R language rules).
                is_registry = (
                    rule_path.startswith(("http://", "https://"))
                    or rule_path == "auto"
                    or (rule_path.startswith(("p/", "r/")) and not rule_path.endswith(".yaml"))
                )
                if is_registry:
                    # Registry/remote paths are used as-is, no file validation needed
                    available_rules.append(rule_path)
                    continue

                # Check all base paths (official + custom rules) for local files
                rule_found = False
                for base_path in base_paths:
                    full_path = os.path.join(base_path, rule_path)
                    if os.path.exists(full_path):
                        # CRITICAL: Convert to absolute path so OpenGrep can find rules
                        # regardless of its working directory (cwd)
                        abs_path = os.path.abspath(full_path)
                        # Pass directories directly to OpenGrep — it handles recursion into
                        # subdirectories natively and skips non-YAML files.  Manual
                        # os.listdir expansion was non-recursive and missed audit/ subtrees.
                        available_rules.append(abs_path)
                        rule_found = True
                        break  # Found in this base path, no need to check others

                if not rule_found:
                    missing_rules.append(f"{language}:{rule_path}")

            # If no local rules found, use registry fallbacks (packaged/production builds)
            if not available_rules and language in registry_fallbacks:
                available_rules = registry_fallbacks[language]
                logger.info(f"Using registry fallback rules for {language}: {available_rules}")

            language_to_rules_map[language] = available_rules
            total_rules += len(available_rules)

        if missing_rules:
            logger.warning(f"Missing rule paths: {missing_rules}")

        logger.info(
            f"opengrep config loaded: {len(extension_to_language_map)} extensions mapped to {len(language_to_rules_map)} languages with {total_rules} total rule sets"
        )

        return ScannerConfig(
            extension_to_language_map,
            language_to_rules_map,
            code_analysis_settings.timeout_seconds,
        )

    except FileNotFoundError as e:
        logger.error(f"scanner configuration file not found: {e}")
        raise HTTPException(
            status_code=500, detail="opengrep configuration file not found"
        )
    except Exception as e:
        logger.exception("Failed to load scanner configuration")
        raise HTTPException(
            status_code=500, detail=f"Failed to load opengrep configuration: {str(e)}"
        )


class CodeScanRequest(BaseModel):
    """Request model for code security scanning

    Attributes:
        target_path: Absolute or relative path to file or directory to scan
        config: Optional scanner config (e.g., 'p/python', 'auto').
                If None, auto-detects language and uses security rules
    """

    target_path: str
    config: Optional[str] = None


class CodeScanResponse(BaseModel):
    """Response model for security scan results

    Attributes:
        success: Whether the scan completed successfully
        data: security scan results (findings, errors, etc.) as JSON dict
        language_detected: Programming language detected from file extension (if auto-detected)
        rules_used: List of rule file paths or config names used in the scan
    """

    success: bool
    data: dict
    language_detected: Optional[str] = None
    rules_used: Optional[List[str]] = None


def run_scan_sync(
    target_path: str,
    config: Optional[str],
    scanner_config: ScannerConfig,
    selected_extensions: Optional[List[str]],
    excluded_dir_list: Optional[List[str]],
    session: SessionDep,
    user_id: str,
) -> Dict[str, Any]:
    """Synchronous security scan execution (simplified from streaming version)"""
    import subprocess

    # Security: Validate and normalize path to prevent traversal attacks
    try:
        target_path = validate_and_normalize_path(target_path)
    except ValueError as e:
        logger.error(f"Path validation failed: {e}")
        raise
    
    # Validate file/directory exists and is accessible
    if not os.path.exists(target_path):
        raise ValueError(f"Target path does not exist: {target_path}")
    if not os.access(target_path, os.R_OK):
        raise ValueError(f"Target path is not readable: {target_path}")

    # Build opengrep command
    _exe = _opengrep_exe()
    scan_args = ["scan", "--json", "--quiet", "--no-git-ignore", "--scan-unknown-extensions"]
    cmd = [_exe] + scan_args

    # Use provided config or auto-detect
    if config:
        cmd.extend(["--config", config])
    else:
        # Check if it is a config file
        config_type = detect_config_type(target_path) if os.path.isfile(target_path) else None
        
        if config_type:
            logger.info(f"Detected config file type: {config_type}")
            rules = get_rules_for_config(config_type)
            for rule in rules:
                cmd.extend(["--config", rule])
        # Auto-detect language and use configured rules
        else:
            # Detect language from file extension or directory scan
            language = None

            # If it's a directory, scan files inside to detect languages
            if os.path.isdir(target_path):
                import glob
                detected_languages = set()

                # Scan files in directory to detect languages
                # Dynamically get all supported extensions from config
                all_extensions = list(scanner_config.extension_to_language_map.keys())
                for ext in all_extensions:
                    pattern = f'*{ext}'
                    found = (
                        glob.glob(os.path.join(target_path, '**', pattern), recursive=True) or
                        glob.glob(os.path.join(target_path, '**', ext), recursive=True)
                    )
                    if found:
                        file_lang = scanner_config.detect_language_from_extension(ext)
                        if file_lang:
                            detected_languages.add(file_lang)

                # If we found languages in the directory, use configured security rules
                if detected_languages:
                    logger.info(f"Detected languages in directory: {detected_languages}")

                    # Get all security rules for detected languages
                    all_rules = []
                    for lang in detected_languages:
                        rules = scanner_config.get_security_rules_for_language(lang)
                        if rules:
                            all_rules.extend(rules)

                    if all_rules:
                        logger.info(f"Using {len(all_rules)} configured security rules for detected languages")
                        # Convert relative config paths to absolute paths
                        # (needed when running OpenGrep from a different working directory)
                        backend_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
                        for rule in all_rules:
                            # Registry paths (p/*, r/*) and URLs should be used as-is.
                            # Local rule paths always end with .yaml — never treat those as
                            # registry rules even if they start with "r/" (e.g. R language rules).
                            is_registry = (
                                rule.startswith(("http://", "https://"))
                                or rule == "auto"
                                or (rule.startswith(("p/", "r/")) and not rule.endswith(".yaml"))
                            )
                            if is_registry:
                                cmd.extend(["--config", rule])
                            # If rule path is relative, convert to absolute from backend directory
                            elif not os.path.isabs(rule):
                                abs_rule = os.path.normpath(os.path.join(backend_dir, rule))
                                cmd.extend(["--config", abs_rule])
                            else:
                                cmd.extend(["--config", rule])
                    else:
                        if getattr(sys, 'frozen', False):
                            logger.warning(f"No rules configured for detected languages, skipping (frozen build)")
                        else:
                            logger.warning(f"No rules configured for detected languages, using auto")
                            cmd.extend(["--config", "auto"])
                else:
                    if getattr(sys, 'frozen', False):
                        logger.warning(f"Could not detect languages in directory {target_path}, skipping (frozen build)")
                    else:
                        logger.warning(f"Could not detect languages in directory {target_path}, using auto")
                        cmd.extend(["--config", "auto"])
            else:
                # Single file - detect from extension
                language = scanner_config.detect_language_from_extension(target_path)

                if language:
                    # Get configured security rules for this language
                    rules = scanner_config.get_security_rules_for_language(language)

                    if rules:
                        logger.info(f"Using configured rules for {language}: {rules}")
                        # Convert relative config paths to absolute paths
                        backend_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
                        for rule in rules:
                            # Registry paths (p/*, r/*) and URLs should be used as-is.
                            # Local rule paths always end with .yaml — never treat those as
                            # registry rules even if they start with "r/" (e.g. R language rules).
                            is_registry = (
                                rule.startswith(("http://", "https://"))
                                or rule == "auto"
                                or (rule.startswith(("p/", "r/")) and not rule.endswith(".yaml"))
                            )
                            if is_registry:
                                cmd.extend(["--config", rule])
                            # If rule path is relative, convert to absolute from backend directory
                            elif not os.path.isabs(rule):
                                abs_rule = os.path.normpath(os.path.join(backend_dir, rule))
                                cmd.extend(["--config", abs_rule])
                            else:
                                cmd.extend(["--config", rule])
                    else:
                        if getattr(sys, 'frozen', False):
                            logger.warning(f"No rules configured for language {language}, skipping (frozen build)")
                        else:
                            logger.warning(f"No rules configured for language {language}, using auto")
                            cmd.extend(["--config", "auto"])
                else:
                    if getattr(sys, 'frozen', False):
                        logger.warning(f"Could not detect language for {target_path}, skipping (frozen build)")
                    else:
                        logger.warning(f"Could not detect language for {target_path}, using auto")
                        cmd.extend(["--config", "auto"])

    # For directories, find all files recursively to bypass OpenGrep's default exclusions (test/, tests/, etc.)
    if os.path.isdir(target_path):
        import glob
        all_files = []

        # Get all files recursively matching the extension filter
        if selected_extensions:
            for ext in selected_extensions:
                pattern = os.path.join(target_path, "**", f"*.{ext}")
                all_files.extend(glob.glob(pattern, recursive=True))
                # Also match pure dotfiles (e.g. .envrc when ext="envrc")
                all_files.extend(glob.glob(os.path.join(target_path, "**", f".{ext}"), recursive=True))
        else:
            # No filter, get all files
            pattern = os.path.join(target_path, "**", "*.*")
            all_files.extend(glob.glob(pattern, recursive=True))
            # Also match pure dotfiles (.env, .envrc, etc.) not captured by *.*
            for ext in scanner_config.extension_to_language_map.keys():
                all_files.extend(glob.glob(os.path.join(target_path, "**", ext), recursive=True))

        # Remove duplicates and filter out directories
        all_files = [f for f in set(all_files) if os.path.isfile(f)]

        # Filter out files in excluded directories
        if excluded_dir_list:
            def is_excluded(file_path: str) -> bool:
                """Check if file is in any excluded directory"""
                path_parts = file_path.replace('\\', '/').split('/')
                for excluded in excluded_dir_list:
                    # Handle glob patterns like *.egg-info
                    if '*' in excluded:
                        for part in path_parts:
                            if fnmatch.fnmatch(part, excluded):
                                return True
                    else:
                        # Exact directory name match
                        if excluded in path_parts:
                            return True
                return False

            original_count = len(all_files)
            all_files = [f for f in all_files if not is_excluded(f)]
            excluded_count = original_count - len(all_files)
            if excluded_count > 0:
                logger.info(f"Excluded {excluded_count} files from scanning based on directory exclusions")

        # Handle different scenarios based on number of files found
        def _run_cmd(extra_args, cwd=None):
            full_cmd = cmd + extra_args
            logger.info(f"Running opengrep command: {full_cmd}")
            return subprocess.run(full_cmd, capture_output=True, text=True,
                                  timeout=code_analysis_settings.timeout_seconds, cwd=cwd)

        if len(all_files) == 0:
            logger.warning(f"No files found in directory {target_path}")
            result = _run_cmd([target_path])
        elif len(all_files) > 10:
            logger.info(f"Found {len(all_files)} files - using directory scan with configured rules")
            result = _run_cmd([target_path])
        else:
            logger.info(f"Found {len(all_files)} files to scan recursively (bypassing opengrep's test/ exclusion)")
            rel_files = [os.path.relpath(f, target_path) for f in all_files]
            logger.info(f"Running opengrep with {len(rel_files)} relative file paths from {target_path}")
            result = _run_cmd(rel_files, cwd=target_path)
    else:
        # Single file
        full_cmd = cmd + [target_path]
        logger.info(f"Running opengrep command (single file): {full_cmd}")
        result = subprocess.run(full_cmd, capture_output=True, text=True,
                                timeout=code_analysis_settings.timeout_seconds)

    # Debug logging
    logger.info(f"opengrep exit code: {result.returncode}")
    logger.info(f"opengrep stdout length: {len(result.stdout) if result.stdout else 0}")
    if result.stderr:
        logger.error(f"opengrep stderr (full): {result.stderr}")

    # Log stdout on error for debugging
    if result.returncode not in [0, 2] and result.stdout:
        logger.error(f"opengrep stdout (exit code {result.returncode}): {result.stdout[:2000]}")

    # Parse results
    scan_results = json.loads(result.stdout) if result.stdout else {}
    findings = scan_results.get("results", [])
    scanned_paths = scan_results.get("paths", {}).get("scanned", [])
    logger.info(f"Parsed scan results: {len(findings)} findings, {len(scanned_paths)} files scanned")

    # Save to database if session provided
    if session:
        # Check if scanning a directory or a single file
        is_directory = os.path.isdir(target_path)

        if is_directory:
            # For directory scans, save individual file records
            scan_id = str(uuid.uuid4())

            # Use helper to enrich findings and create standardized structure
            scan_results["enriched_findings"] = enrich_findings(findings, session=session, user_id=user_id)
            save_directory_scan_results(
                session, scan_results, scan_id, target_path, user_id=user_id
            )
        else:
            # For single file scans, use shared helper for consistent structure
            scan_data = create_scan_data(findings, scan_results, session=session, user_id=user_id)
            # For individual files, use parent directory as project_path
            project_path = os.path.dirname(target_path) if os.path.isfile(target_path) else target_path
            save_code_scan_to_db(session, target_path, scan_data, user_id=user_id, project_path=project_path)

    return scan_results


@router.post("/code/scan")
async def code_scan(
    target_path: str,
    config: Optional[str] = None,
    extensions: Optional[str] = None,
    excluded_dirs: Optional[str] = None,
    business_criticality: Optional[str] = "medium",
    compliance_requirements: Optional[str] = None,
    dry_run: bool = False,
    scanner_config: ScannerConfig = Depends(load_scanner_config),
    session: SessionDep = None,
    user: User = Depends(get_current_user),
):
    """
    Run OpenGrep security scan and return results

    Performs static analysis security scanning using OpenGrep.
    If no config is provided, automatically detects the programming language from file
    extension and applies appropriate security rules.

    Args:
        target_path: Path to file or directory to scan
        config: Optional scanner config (e.g., 'p/python', 'auto').
               If None, auto-detects language and uses security rules
        extensions: Optional comma-separated list of file extensions to scan (e.g., 'py,ts,js')
        excluded_dirs: Optional comma-separated list of directories to exclude (e.g., 'node_modules,*.egg-info')
        business_criticality: Business criticality level ('low', 'medium', 'high', 'critical')
        compliance_requirements: Comma-separated compliance frameworks (e.g., 'fips_140,fedramp,hipaa')
        scanner_config: Injected scanner configuration (auto-loaded)
        session: Database session for storing results

    Returns:
        Scan results with findings
    """
    # Rate limiting
    allowed, retry_after = scan_rate_limiter.is_allowed(user.id)
    if not allowed:
        raise HTTPException(
            status_code=429,
            detail=f"Rate limit exceeded. Retry after {retry_after} seconds",
            headers={"Retry-After": str(retry_after)},
        )

    # License tier limit check
    license_info = LicenseValidator.validate()
    if session:
        enforce_scan_limits(session, user.id, license_info.get("license_type", "trial"))

    try:
        # Validate inputs
        try:
            config = validate_config(config)
            
            # Validate business criticality
            if business_criticality:
                business_criticality = business_criticality.lower()
                if business_criticality not in ("low", "medium", "high", "critical"):
                    raise ValueError(f"Invalid business_criticality: {business_criticality}")
            
            # Parse and validate compliance requirements
            compliance_list = []
            if compliance_requirements:
                allowed_compliance = {"fips_140", "fedramp", "hipaa", "pci_dss", "gdpr", "sox", "common_criteria", "itar"}
                compliance_list = [c.strip().lower() for c in compliance_requirements.split(",") if c.strip()]
                for req in compliance_list:
                    if req not in allowed_compliance:
                        raise ValueError(f"Invalid compliance requirement: {req}")
        except ValueError as e:
            raise HTTPException(status_code=400, detail=str(e))
        
        # Parse extensions parameter if provided
        selected_extensions = None
        if extensions:
            try:
                ext_list = [ext.strip() for ext in extensions.split(",") if ext.strip()]
                selected_extensions = sanitize_extensions(ext_list)
                logger.info(f"Extension filter enabled: {selected_extensions}")
            except ValueError as e:
                raise HTTPException(status_code=400, detail=str(e))

        # Parse excluded directories
        excluded_dir_list = None
        if excluded_dirs:
            try:
                dir_list = [d.strip() for d in excluded_dirs.split(",") if d.strip()]
                excluded_dir_list = sanitize_excluded_dirs(dir_list)
                logger.info(f"Directory exclusions enabled: {excluded_dir_list}")
            except ValueError as e:
                raise HTTPException(status_code=400, detail=str(e))

        logger.info(f"Starting opengrep scan for: {target_path} (criticality={business_criticality}, compliance={compliance_list})")

        # Run the scan (simplified from streaming version)
        results = await run_in_threadpool(
            run_scan_sync,
            target_path,
            config,
            scanner_config,
            selected_extensions,
            excluded_dir_list,
            None if dry_run else session,
            user.id
        )
        
        # Add business impact metadata to results
        if "enriched_findings" not in results:
            results["enriched_findings"] = enrich_findings(results.get("results", []), session=session, user_id=user.id)
        
        results["business_context"] = {
            "criticality": business_criticality,
            "compliance": compliance_list,
        }

        return {"success": True, "results": results}
    except HTTPException:
        raise
    except ValueError as e:
        logger.error(f"Validation error: {e}")
        raise HTTPException(status_code=400, detail=str(e))
    except Exception as e:
        logger.exception(f"Failed to run opengrep scan for {target_path}")
        safe_msg = sanitize_error_message(e, "opengrep scan")
        raise HTTPException(status_code=500, detail=f"Scan failed: {safe_msg}")

async def get_user_from_token(
    token: Optional[str] = Query(None),
    user_manager = Depends(get_user_manager),
) -> Optional[User]:
    """
    Custom authentication dependency for streaming endpoints.
    Accepts token from query parameter since SSE doesn't reliably support headers.
    Returns None if no token provided (for desktop app without auth).

    Args:
        token: JWT token from query parameter
        user_manager: User manager dependency

    Returns:
        Authenticated User object or None if no token

    Raises:
        HTTPException: If token is invalid
    """
    if not token:
        return None

    try:
        # Get JWT strategy and decode token
        strategy = auth_backend.get_strategy()
        user = await strategy.read_token(token, user_manager)

        if not user or not user.is_active:
            raise HTTPException(status_code=401, detail="Invalid or expired token")

        return user
    except Exception as e:
        logger.error(f"Token authentication failed: {e}")
        raise HTTPException(status_code=401, detail="Invalid or expired token")


@router.post("/code/scan/stream")
async def code_scan_stream(
    request: Request,
    target_path: str,
    config: Optional[str] = None,
    extensions: Optional[str] = None,
    excluded_dirs: Optional[str] = None,
    business_criticality: Optional[str] = "medium",
    compliance_requirements: Optional[str] = None,
    llm_provider: Optional[str] = None,
    llm_base_url: Optional[str] = None,
    llm_model: Optional[str] = None,
    scanner_config: ScannerConfig = Depends(load_scanner_config),
    session: SessionDep = None,
    user: Optional[User] = Depends(get_user_from_token),
):
    """
    Run OpenGrep security scan with streaming results (Server-Sent Events)

    This endpoint streams scan results as they are found, perfect for large
    directories. No timeouts - results stream continuously until complete.

    Args:
        target_path: Path to file or directory to scan
        config: Optional scanner config
        extensions: Optional comma-separated list of file extensions
        excluded_dirs: Optional comma-separated list of directories to exclude
        business_criticality: Business criticality level ('low', 'medium', 'high', 'critical')
        compliance_requirements: Comma-separated compliance frameworks
        scanner_config: Injected scanner configuration
        session: Database session
        user: Current authenticated user

    Returns:
        StreamingResponse with Server-Sent Events (SSE) format

    SSE Event Types:
        - progress: {"files_scanned": N, "total_files": M, "findings": X}
        - finding: {...finding object...}
        - complete: {"total_findings": N, "scan_id": "uuid"}
        - error: {"message": "error description"}
    """
    # Rate limiting (skip if no user)
    if user:
        allowed, retry_after = scan_rate_limiter.is_allowed(user.id)
        if not allowed:
            async def rate_limit_error():
                yield f"data: {json.dumps({'type': 'error', 'message': f'Rate limit exceeded. Retry after {retry_after} seconds'})}\n\n"
            return StreamingResponse(
                rate_limit_error(),
                media_type="text/event-stream",
                status_code=429,
                headers={"Retry-After": str(retry_after)},
            )

    # License tier limit check (skip if no user)
    if user and session:
        license_info = LicenseValidator.validate()
        enforce_scan_limits(session, user.id, license_info.get("license_type", "trial"))

    async def event_generator():
        # Initialize variables for partial result recovery
        is_directory = False
        all_findings = []
        all_files_preview = []
        files_scanned = 0
        total_files = 0
        scan_id = str(uuid.uuid4())
        business_criticality_validated = business_criticality or "medium"
        compliance_list = []
        
        # Build LLM config if provided
        llm_config = None
        llm_api_key = request.headers.get("X-LLM-Key")
        if llm_provider and llm_api_key:
            llm_config = {
                "provider": llm_provider,
                "apiKey": llm_api_key,
                "baseUrl": llm_base_url,
                "model": llm_model,
            }

        try:
            # Validate inputs
            try:
                config_validated = validate_config(config)
                
                # Validate business criticality
                if business_criticality:
                    business_criticality_validated = business_criticality.lower()
                    if business_criticality_validated not in ("low", "medium", "high", "critical"):
                        raise ValueError(f"Invalid business_criticality: {business_criticality_validated}")
                
                # Parse and validate compliance requirements
                if compliance_requirements:
                    allowed_compliance = {"fips_140", "fedramp", "hipaa", "pci_dss", "gdpr", "sox", "common_criteria", "itar"}
                    compliance_list = [c.strip().lower() for c in compliance_requirements.split(",") if c.strip()]
                    for req in compliance_list:
                        if req not in allowed_compliance:
                            raise ValueError(f"Invalid compliance requirement: {req}")
            except ValueError as e:
                yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
                return
            
            # Parse extensions
            selected_extensions = None
            if extensions:
                try:
                    ext_list = [ext.strip() for ext in extensions.split(",") if ext.strip()]
                    selected_extensions = sanitize_extensions(ext_list)
                except ValueError as e:
                    yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
                    return

            # Parse excluded directories
            excluded_dir_list = []
            if excluded_dirs:
                try:
                    dir_list = [d.strip() for d in excluded_dirs.split(",") if d.strip()]
                    excluded_dir_list = sanitize_excluded_dirs(dir_list)
                except ValueError as e:
                    yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n"
                    return

            # Security: Validate path
            target_path_validated = validate_and_normalize_path(target_path)

            # Check if directory or file
            is_directory = os.path.isdir(target_path_validated)

            if is_directory:
                # HYBRID APPROACH: Smart batching for speed + real-time progress
                # - Small scans (<20 files): Run once (fastest)
                # - Medium scans (20-100 files): Batch by 10-15 files (good balance)
                # - Large scans (100+ files): Batch by 20-30 files (still fast, shows progress)

                # Count files FIRST to show initial progress
                import glob
                all_files_preview = []
                if selected_extensions:
                    for ext in selected_extensions:
                        pattern = os.path.join(target_path_validated, "**", f"*.{ext}")
                        all_files_preview.extend(glob.glob(pattern, recursive=True))
                        # Also match pure dotfiles (e.g. .envrc when ext="envrc")
                        all_files_preview.extend(glob.glob(os.path.join(target_path_validated, "**", f".{ext}"), recursive=True))
                else:
                    pattern = os.path.join(target_path_validated, "**", "*.*")
                    all_files_preview.extend(glob.glob(pattern, recursive=True))
                    # Also match pure dotfiles (.env, .envrc, etc.) not captured by *.*
                    for ext in scanner_config.extension_to_language_map.keys():
                        all_files_preview.extend(glob.glob(os.path.join(target_path_validated, "**", ext), recursive=True))

                all_files_preview = [f for f in set(all_files_preview) if os.path.isfile(f)]

                # Filter out files in excluded directories
                if excluded_dir_list:
                    def is_excluded(file_path: str) -> bool:
                        """Check if file is in any excluded directory"""
                        path_parts = file_path.replace('\\', '/').split('/')
                        for excluded in excluded_dir_list:
                            # Handle glob patterns like *.egg-info
                            if '*' in excluded:
                                for part in path_parts:
                                    if fnmatch.fnmatch(part, excluded):
                                        return True
                            else:
                                # Exact directory name match
                                if excluded in path_parts:
                                    return True
                        return False

                    original_count = len(all_files_preview)
                    all_files_preview = [f for f in all_files_preview if not is_excluded(f)]
                    excluded_count = original_count - len(all_files_preview)
                    if excluded_count > 0:
                        logger.info(f"Excluded {excluded_count} files from scanning based on directory exclusions")

                total_files = len(all_files_preview)

                if total_files == 0:
                    yield f"data: {json.dumps({'type': 'error', 'message': 'No files found in directory'})}\n\n"
                    return

                scan_id = str(uuid.uuid4())

                # Send scan_id to frontend so it can request cancellation
                yield f"data: {json.dumps({'type': 'scan_started', 'scan_id': scan_id, 'total_files': total_files})}\n\n"

                # Send initial progress with file count
                yield f"data: {json.dumps({'type': 'progress', 'files_scanned': 0, 'total_files': total_files, 'findings': 0, 'status': 'scanning'})}\n\n"

                # Adaptive batching — keep batches small enough that cancel
                # checks fire frequently (checked after every batch).
                if total_files <= 10:
                    batch_size = 5
                elif total_files <= 50:
                    batch_size = 10
                elif total_files <= 100:
                    batch_size = 15
                else:
                    batch_size = 25

                all_findings = []
                files_scanned = 0

                # Group files by language to avoid mixing rules
                from collections import defaultdict
                files_by_language = defaultdict(list)
                for file_path in all_files_preview:
                    language = scanner_config.detect_language_from_extension(file_path)
                    if language:
                        files_by_language[language].append(file_path)
                    else:
                        files_by_language['unknown'].append(file_path)

                # ── Incremental scan cache ────────────────────────────────────
                from app.code_analyzer.scan_cache import (
                    ScanCache,
                    compute_language_rule_fingerprint,
                    compute_supplementary_fingerprint,
                    compute_file_hash,
                    split_cached_vs_dirty,
                    queue_batch_cache_writes,
                )
                scan_cache = ScanCache(session) if session else None

                # Pre-compute per-language rule fingerprints (one git/hash call per language)
                lang_rule_fingerprints: Dict[str, str] = {}
                if scan_cache:
                    for lang in files_by_language:
                        if lang == 'unknown':
                            continue
                        rules = scanner_config.get_security_rules_for_language(lang)
                        lang_rule_fingerprints[lang] = compute_language_rule_fingerprint(
                            lang, rules, "all"
                        )
                    logger.info(
                        f"Incremental cache: fingerprints computed for {len(lang_rule_fingerprints)} languages"
                    )

                # Process each language group separately
                for language, lang_files in files_by_language.items():
                    if language == 'unknown':
                        continue  # Skip unknown file types

                    # Get rules for this specific language
                    language_rules = scanner_config.get_security_rules_for_language(language)
                    if not language_rules:
                        logger.warning(f"No rules found for language: {language}, skipping {len(lang_files)} files")
                        continue

                    # ── Cache split ───────────────────────────────────────────
                    # Separate files whose cached results are still valid from files that need scanning.
                    rule_fp = lang_rule_fingerprints.get(language, "")
                    if scan_cache and rule_fp:
                        cached_files, dirty_files, cached_findings = split_cached_vs_dirty(
                            lang_files, target_path_validated, language, rule_fp, scan_cache
                        )
                        if cached_files:
                            all_findings.extend(cached_findings)
                            files_scanned += len(cached_files)
                            logger.info(
                                f"Cache hit: {len(cached_files)}/{len(lang_files)} {language} files "
                                f"served from cache ({len(cached_findings)} findings)"
                            )
                            # Send progress so the UI bar advances for cached files
                            try:
                                yield f"data: {json.dumps({'type': 'progress', 'files_scanned': files_scanned, 'total_files': total_files, 'findings': len(all_findings), 'status': 'scanning'})}\n\n"
                            except Exception:
                                pass
                        if not dirty_files:
                            continue  # Entire language group served from cache
                    else:
                        dirty_files = lang_files

                    # Batch within each language group (dirty files only)
                    for i in range(0, len(dirty_files), batch_size):
                        # Check if scan was cancelled via cancel endpoint or client disconnected
                        cancelled = _is_scan_cancelled(scan_id)
                        disconnected = await request.is_disconnected()
                        if cancelled or disconnected:
                            reason = "cancel request" if cancelled else "client disconnect"
                            logger.info(f"Scan stopped ({reason}). Saving partial results: {len(all_findings)} findings from {files_scanned}/{total_files} files")
                            # Flush any pending cache writes before stopping
                            if scan_cache:
                                scan_cache.flush_pending()
                            # Save partial results before stopping (even 0 findings — shows in history)
                            if session:
                                try:
                                    scan_results = {"results": all_findings, "paths": {"scanned": all_files_preview[:files_scanned]}}
                                    enriched = enrich_findings(all_findings, session=session, user_id="local")
                                    scan_results["enriched_findings"] = enriched
                                    await run_in_threadpool(
                                        save_directory_scan_results,
                                        session,
                                        scan_results,
                                        scan_id,
                                        target_path_validated,
                                        user_id="local",
                                        is_partial=True,
                                        partial_reason="cancelled"
                                    )
                                    logger.info(f"Partial results saved after {reason}")
                                    # Send partial_complete if client is still connected (cancel endpoint case)
                                    if cancelled and not disconnected:
                                        partial_files_with_issues = len({f.get("path") or f.get("location", {}).get("path", "") for f in all_findings if f.get("path") or f.get("location", {}).get("path", "")})
                                        yield f"data: {json.dumps({'type': 'partial_complete', 'total_findings': len(all_findings), 'scan_id': scan_id, 'files_scanned': files_scanned, 'total_files': total_files, 'files_with_issues': partial_files_with_issues})}\n\n"
                                except Exception as save_error:
                                    logger.exception(f"Failed to save partial results after {reason}")
                            _cleanup_scan(scan_id)
                            return  # Stop scanning

                        batch_files = dirty_files[i:i+batch_size]

                        # Run OpenGrep on this batch with language-specific rules
                        batch_result = await run_in_threadpool(
                            scan_files_batch_with_rules,
                            batch_files,
                            language_rules,
                            language,
                            scanner_config,
                            target_path_validated
                        )

                        batch_findings = batch_result.get("findings", [])
                        all_findings.extend(batch_findings)
                        files_scanned += len(batch_files)

                        # Queue cache writes for this batch (flushed after language group)
                        if scan_cache and rule_fp:
                            queue_batch_cache_writes(
                                batch_files, batch_findings,
                                target_path_validated, language, rule_fp, scan_cache
                            )

                        # ── Post-batch cancel check ──────────────────────────
                        # The OpenGrep subprocess may have blocked for seconds.
                        # Check if the user cancelled while it was running.
                        if _is_scan_cancelled(scan_id):
                            logger.info(f"Scan cancelled after batch. Saving partial results: {len(all_findings)} findings from {files_scanned}/{total_files} files")
                            if scan_cache:
                                scan_cache.flush_pending()
                            if session:
                                try:
                                    scan_results = {"results": all_findings, "paths": {"scanned": all_files_preview[:files_scanned]}}
                                    enriched = enrich_findings(all_findings, session=session, user_id="local")
                                    scan_results["enriched_findings"] = enriched
                                    await run_in_threadpool(
                                        save_directory_scan_results,
                                        session,
                                        scan_results,
                                        scan_id,
                                        target_path_validated,
                                        user_id="local",
                                        is_partial=True,
                                        partial_reason="cancelled"
                                    )
                                    logger.info(f"Partial results saved after cancel (post-batch)")
                                    partial_files_with_issues = len({f.get("path") or f.get("location", {}).get("path", "") for f in all_findings if f.get("path") or f.get("location", {}).get("path", "")})
                                    yield f"data: {json.dumps({'type': 'partial_complete', 'total_findings': len(all_findings), 'scan_id': scan_id, 'files_scanned': files_scanned, 'total_files': total_files, 'files_with_issues': partial_files_with_issues})}\n\n"
                                except Exception as save_error:
                                    logger.exception(f"Failed to save partial results after cancel (post-batch)")
                            _cleanup_scan(scan_id)
                            return  # Stop scanning

                        # Send progress update after each batch (show RAW count during scan, filtering happens at end)
                        try:
                            yield f"data: {json.dumps({'type': 'progress', 'files_scanned': files_scanned, 'total_files': total_files, 'findings': len(all_findings), 'status': 'scanning'})}\n\n"
                        except Exception as e:
                            # Client likely disconnected, trigger cleanup
                            logger.info(f"Failed to send progress update, client likely disconnected: {e}")
                            raise

                        # Small delay to allow client to process
                        await asyncio.sleep(0.01)

                    # Flush queued cache writes for this language group
                    if scan_cache:
                        scan_cache.flush_pending()

                if not all_findings and total_files == 0:
                    yield f"data: {json.dumps({'type': 'error', 'message': 'Scan failed - check logs'})}\n\n"
                    return

                # ── Final cancel check ──────────────────────────────
                # All batches finished, but user may have clicked cancel
                # during the last batch or enrichment window.  Catch it
                # here so the scan is still marked partial.
                if _is_scan_cancelled(scan_id):
                    logger.info(f"Scan cancelled after all batches completed. Marking as partial: {len(all_findings)} findings from {files_scanned}/{total_files} files")
                    if session:
                        try:
                            scan_results = {"results": all_findings, "paths": {"scanned": all_files_preview}}
                            enriched = enrich_findings(all_findings, session=session, user_id="local")
                            scan_results["enriched_findings"] = enriched
                            scan_results["files_with_issues"] = len({f.get("path") or f.get("location", {}).get("path", "") for f in enriched if f.get("path") or f.get("location", {}).get("path", "")})
                            scan_results["business_context"] = {
                                "criticality": business_criticality_validated,
                                "compliance": compliance_list,
                            }
                            await run_in_threadpool(
                                save_directory_scan_results,
                                session,
                                scan_results,
                                scan_id,
                                target_path_validated,
                                user_id="local",
                                is_partial=True,
                                partial_reason="cancelled"
                            )
                            logger.info(f"Partial results saved (cancel caught at completion)")
                            for finding in enriched:
                                yield f"data: {json.dumps({'type': 'finding', 'data': finding})}\n\n"
                                await asyncio.sleep(0.001)
                            partial_files_with_issues = scan_results["files_with_issues"]
                            yield f"data: {json.dumps({'type': 'partial_complete', 'total_findings': len(enriched), 'scan_id': scan_id, 'files_scanned': files_scanned, 'total_files': total_files, 'files_with_issues': partial_files_with_issues})}\n\n"
                        except Exception as save_error:
                            logger.exception(f"Failed to save partial results (cancel at completion)")
                    _cleanup_scan(scan_id)
                    return

                # Send scan completion progress
                yield f"data: {json.dumps({'type': 'progress', 'files_scanned': total_files, 'total_files': total_files, 'findings': len(all_findings), 'status': 'complete'})}\n\n"

                # Run supplementary custom scanners on Java and Python files
                # These complement OpenGrep with regex patterns for specific libraries
                # Each supplementary scanner has its own cache key (language = 'java_bouncycastle' etc.)
                for supp_language, supp_scanner_lang, supp_name in [
                    ('java',   'java_bouncycastle',  'bouncycastle'),
                    ('python', 'python_paramiko',    'paramiko'),
                ]:
                    supp_files = list(files_by_language.get(supp_language, []))
                    if not supp_files:
                        continue

                    supp_fp = compute_supplementary_fingerprint(supp_language) if scan_cache else ""
                    supp_dirty = supp_files
                    supp_cached_findings: List[Dict[str, Any]] = []

                    if scan_cache and supp_fp:
                        _, supp_dirty, supp_cached_findings = split_cached_vs_dirty(
                            supp_files, target_path_validated, supp_scanner_lang, supp_fp, scan_cache
                        )
                        if supp_cached_findings:
                            all_findings.extend(supp_cached_findings)
                            logger.info(
                                f"{supp_name.capitalize()} supplementary cache: "
                                f"{len(supp_files) - len(supp_dirty)}/{len(supp_files)} files from cache "
                                f"({len(supp_cached_findings)} findings)"
                            )

                    if supp_dirty:
                        supp_result = await run_in_threadpool(
                            scan_files_batch_with_rules, supp_dirty, [], supp_name,
                            scanner_config, target_path_validated
                        )
                        supp_findings = supp_result.get('findings', [])
                        if supp_findings:
                            all_findings.extend(supp_findings)
                        logger.info(
                            f"{supp_name.capitalize()} supplementary scanner: "
                            f"{len(supp_findings)} findings from {len(supp_dirty)} files"
                        )
                        # Cache results for dirty files
                        if scan_cache and supp_fp:
                            queue_batch_cache_writes(
                                supp_dirty, supp_findings,
                                target_path_validated, supp_scanner_lang, supp_fp, scan_cache
                            )
                            scan_cache.flush_pending()

                # Remove cache entries for deleted files (only on a complete non-partial scan)
                if scan_cache:
                    seen_norm_paths = {os.path.normcase(f) for f in all_files_preview}
                    scan_cache.delete_stale_entries(target_path_validated, seen_norm_paths)

                # Enrich findings BEFORE streaming to frontend
                enriched_findings = enrich_findings(all_findings, session=session, user_id="local")
                
                # Stream each enriched finding
                for finding in enriched_findings:
                    yield f"data: {json.dumps({'type': 'finding', 'data': finding})}\n\n"
                    # Small delay to prevent overwhelming the client
                    await asyncio.sleep(0.001)

                # Compute files_with_issues BEFORE saving to DB
                files_with_issues = len({f.get("path") or f.get("location", {}).get("path", "") for f in enriched_findings if f.get("path") or f.get("location", {}).get("path", "")})

                # Save to database if session provided (save even with 0 findings so scan appears in history)
                if session:
                    try:
                        scan_results = {"results": all_findings, "paths": {"scanned": all_files_preview}}
                        scan_results["enriched_findings"] = enriched_findings
                        scan_results["files_with_issues"] = files_with_issues
                        scan_results["business_context"] = {
                            "criticality": business_criticality_validated,
                            "compliance": compliance_list,
                        }
                        await run_in_threadpool(
                            save_directory_scan_results,
                            session,
                            scan_results,
                            scan_id,
                            target_path_validated,
                            user_id="local",
                        )
                    except Exception as db_error:
                        logger.exception(f"Failed to save scan results to database")
                        yield f"data: {json.dumps({'type': 'warning', 'message': 'Results not saved to database'})}\n\n"
                # ── Post-save cancel check ───────────────────────────
                # The user may have clicked cancel while findings were
                # being streamed or the DB save was running.  If so,
                # retroactively mark the just-saved record as partial
                # and send partial_complete instead of complete.
                if _is_scan_cancelled(scan_id):
                    logger.info(f"Cancel detected after DB save — retroactively marking scan {scan_id} as partial")
                    if session:
                        try:
                            from sqlmodel import select as _sel
                            stmt = _sel(ScanRecord).where(
                                ScanRecord.scan_type == ScanType.CODE_ANALYSIS
                            ).order_by(ScanRecord.timestamp.desc()).limit(50)  # type: ignore[arg-type]
                            records = session.exec(stmt).all()
                            for record in records:
                                sd = record.scan_data or {}
                                if sd.get("scan_id") == scan_id:
                                    sd["is_partial"] = True
                                    sd["partial_reason"] = "cancelled"
                                    record.scan_data = sd
                                    flag_modified(record, "scan_data")
                                    session.add(record)
                                    session.commit()
                                    logger.info(f"Retroactively set is_partial on record {record.id}")
                                    break
                        except Exception as retro_err:
                            logger.warning(f"Failed to retroactively mark as partial: {retro_err}")
                    _cleanup_scan(scan_id)
                    yield f"data: {json.dumps({'type': 'partial_complete', 'total_findings': len(enriched_findings), 'scan_id': scan_id, 'files_scanned': total_files, 'total_files': total_files, 'files_with_issues': files_with_issues})}\n\n"
                else:
                    _cleanup_scan(scan_id)
                    yield f"data: {json.dumps({'type': 'complete', 'total_findings': len(enriched_findings), 'scan_id': scan_id, 'files_scanned': total_files, 'files_with_issues': files_with_issues, 'business_context': {'criticality': business_criticality_validated, 'compliance': compliance_list}})}\n\n"

            else:
                # Single file - use existing synchronous scan
                results = await run_in_threadpool(
                    run_scan_sync,
                    target_path_validated,
                    config_validated,
                    scanner_config,
                    selected_extensions,
                    None,  # excluded_dir_list
                    session,
                    user.id
                )

                # Yield all findings (filtering is handled by frontend)
                findings = results.get("enriched_findings", [])
                for finding in findings:
                    yield f"data: {json.dumps({'type': 'finding', 'data': finding})}\n\n"

                yield f"data: {json.dumps({'type': 'complete', 'total_findings': len(findings), 'files_scanned': 1})}\n\n"

        except Exception as e:
            logger.exception(f"Streaming scan failed for {target_path}: {type(e).__name__}: {e}")

            # Save partial results if we have any findings before the error
            logger.info(f"Attempting to save partial results: is_directory={is_directory}, findings={len(all_findings)}, session={session is not None}")
            if is_directory and all_findings and session:
                try:
                    logger.info(f"Saving {len(all_findings)} partial results from failed scan")
                    # CRITICAL FIX: Enrich findings before streaming
                    enriched_findings = enrich_findings(all_findings, session=session, user_id="local")

                    # Stream the enriched findings we collected before failure
                    for finding in enriched_findings:
                        yield f"data: {json.dumps({'type': 'finding', 'data': finding})}\n\n"
                        await asyncio.sleep(0.001)

                    # Save partial results to database
                    scan_results = {"results": all_findings, "paths": {"scanned": all_files_preview[:files_scanned]}}
                    scan_results["enriched_findings"] = enriched_findings
                    await run_in_threadpool(
                        save_directory_scan_results,
                        session,
                        scan_results,
                        scan_id,
                        target_path_validated,
                        user_id="local",
                        is_partial=True,
                        partial_reason="error"
                    )
                    logger.info(f"Partial scan results saved: {len(all_findings)} findings from {files_scanned}/{total_files} files")
                    partial_files_with_issues = len({f.get("path") or f.get("location", {}).get("path", "") for f in all_findings if f.get("path") or f.get("location", {}).get("path", "")})
                    yield f"data: {json.dumps({'type': 'partial_complete', 'total_findings': len(all_findings), 'scan_id': scan_id, 'files_scanned': files_scanned, 'total_files': total_files, 'files_with_issues': partial_files_with_issues})}\n\n"
                except Exception as save_error:
                    logger.exception(f"Failed to save partial results")

            # Send error message
            error_msg = sanitize_error_message(e, "Streaming scan")
            if all_findings:
                error_msg += f" ({len(all_findings)} findings from {files_scanned}/{total_files} files saved)"
            yield f"data: {json.dumps({'type': 'error', 'message': error_msg})}\n\n"

    return StreamingResponse(
        event_generator(),
        media_type="text/event-stream",
        headers={
            "Cache-Control": "no-cache",
            "Connection": "keep-alive",
            "X-Accel-Buffering": "no",  # Disable nginx buffering
        }
    )


def scan_files_batch_with_rules(
    files: List[str],
    rules: List[str],
    language: str,
    scanner_config: ScannerConfig,
    base_path: str
) -> Dict[str, Any]:
    """
    Scan a batch of files with pre-computed language-specific rules.
    This avoids re-detecting languages and mixing rules from multiple languages.

    Args:
        files: List of file paths to scan (all same language)
        rules: List of rule paths for this specific language
        language: The language of the files in this batch
        scanner_config: scanner configuration object
        base_path: Base directory path

    Returns:
        Dict with 'findings' and 'files_scanned' keys
    """
    import subprocess

    if not files:
        return {"findings": [], "files_scanned": []}

    # dotenv (.env) files: the generic/secrets/security registry rule triggers OpenGrep's
    # shebang/script detection on lines starting with '#', which crashes with
    # "Failed to canonicalize script path" on Windows frozen builds.
    # Drop the generic registry rule; keep only local custom rules (e.g. weak-crypto-config.yaml).
    if language == 'dotenv':
        rules = [r for r in rules if not r.startswith(('p/', 'r/', 'generic/', 'auto'))]
        if not rules:
            logger.info(f"Skipping opengrep for dotenv files (no safe rules remaining): {len(files)} files")
            return {"findings": [], "files_scanned": files}
        logger.info(f"dotenv: running {len(rules)} local rule(s) only (generic rules excluded): {len(files)} files")

    if language == 'sql':
        scanner = SQLCryptoScanner()
        all_findings = []
        for file_path in files:
            findings = scanner.scan_file(file_path)
            formatted = scanner.format_findings(findings)
            all_findings.extend(formatted.get('results', []))
        logger.info(f"SQL custom scanner: {len(all_findings)} findings from {len(files)} files")
        return {"findings": all_findings, "files_scanned": files}
    
    elif language == 'html':
        scanner = HTMLCryptoScanner()
        all_findings = []
        for file_path in files:
            findings = scanner.scan_file(file_path)
            formatted = scanner.format_findings(findings)
            all_findings.extend(formatted.get('results', []))
        logger.info(f"HTML custom scanner: {len(all_findings)} findings from {len(files)} files")
        return {"findings": all_findings, "files_scanned": files}
    
    elif language == 'apex':
        scanner = ApexCryptoScanner()
        all_findings = []
        for file_path in files:
            findings = scanner.scan_file(file_path)
            formatted = scanner.format_findings(findings)
            all_findings.extend(formatted.get('results', []))
        logger.info(f"Apex custom scanner: {len(all_findings)} findings from {len(files)} files")
        return {"findings": all_findings, "files_scanned": files}
    
    elif language == 'elixir':
        scanner = ElixirCryptoScanner()
        all_findings = []
        for file_path in files:
            findings = scanner.scan_file(file_path)
            formatted = scanner.format_findings(findings)
            all_findings.extend(formatted.get('results', []))
        logger.info(f"Elixir custom scanner: {len(all_findings)} findings from {len(files)} files")
        return {"findings": all_findings, "files_scanned": files}

    elif language == 'certificate':
        scanner = CertificateCryptoScanner()
        all_findings = []
        for file_path in files:
            findings = scanner.scan_file(file_path)
            formatted = scanner.format_findings(findings, file_path)
            all_findings.extend(formatted.get('results', []))
        logger.info(f"Certificate custom scanner: {len(all_findings)} findings from {len(files)} files")
        return {"findings": all_findings, "files_scanned": files}

    elif language == 'bouncycastle':
        scanner = BouncyCastleCryptoScanner()
        all_findings = []
        for file_path in files:
            findings = scanner.scan_file(file_path)
            formatted = scanner.format_findings(findings)
            all_findings.extend(formatted.get('results', []))
        logger.info(f"BouncyCastle custom scanner: {len(all_findings)} findings from {len(files)} files")
        return {"findings": all_findings, "files_scanned": files}

    elif language == 'paramiko':
        scanner = ParamikoCryptoScanner()
        all_findings = []
        for file_path in files:
            findings = scanner.scan_file(file_path)
            formatted = scanner.format_findings(findings)
            all_findings.extend(formatted.get('results', []))
        logger.info(f"Paramiko custom scanner: {len(all_findings)} findings from {len(files)} files")
        return {"findings": all_findings, "files_scanned": files}

    if not rules:
        logger.warning(f"No rules provided for {language} batch, skipping {len(files)} files")
        return {"findings": [], "files_scanned": []}

    # Build OpenGrep args (without the executable prefix — reused for in-process fallback)
    scan_args = ["scan", "--json", "--quiet", "--no-git-ignore", "--scan-unknown-extensions"]

    # Add all rule paths
    backend_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
    for rule in rules:
        # Registry paths (p/*, r/*) and URLs should be used as-is.
        # Local rule paths always end with .yaml — never treat those as
        # registry rules even if they start with "r/" (e.g. R language rules).
        is_registry = (
            rule.startswith(("http://", "https://"))
            or rule == "auto"
            or (rule.startswith(("p/", "r/")) and not rule.endswith(".yaml"))
        )
        if is_registry:
            scan_args.extend(["--config", rule])
        elif not os.path.isabs(rule):
            # Convert relative file paths to absolute
            abs_rule = os.path.normpath(os.path.join(backend_dir, rule))
            scan_args.extend(["--config", abs_rule])
        else:
            scan_args.extend(["--config", rule])

    logger.info(f"Batch scan for {language}: {len(files)} files with {len(rules)} rule paths")

    # Log sample paths for debugging
    if len(rules) <= 3:
        logger.info(f"  Rule paths: {rules}")
    else:
        logger.info(f"  Sample rule paths: {rules[:3]}")

    # Add all files to scan
    scan_args.extend(files)

    # Log command for debugging
    cmd_str = " ".join(scan_args)
    if len(cmd_str) > 500:
        logger.info(f"Running opengrep: {cmd_str[:500]}... (truncated)")
    else:
        logger.info(f"Running opengrep: {cmd_str}")

    # Run opengrep
    _t0 = time.perf_counter()
    _scan_env = {**os.environ, "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8"}
    try:
        cmd = [_opengrep_exe(), *scan_args]
        result = subprocess.run(
            cmd, capture_output=True, text=True, encoding="utf-8", errors="replace",
            cwd=base_path, timeout=600, env=_scan_env
        )
    except subprocess.TimeoutExpired:
        logger.error(f"opengrep batch scan timed out after 600s for {language}")
        return {"findings": [], "files_scanned": files, "timeout": True}

    _duration_ms = round((time.perf_counter() - _t0) * 1000)
    stdout_len = len(result.stdout) if result.stdout else 0
    stderr_len = len(result.stderr) if result.stderr else 0
    logger.info(
        f"opengrep exit code: {result.returncode}, stdout length: {stdout_len}, "
        f"stderr length: {stderr_len}, duration_ms: {_duration_ms}"
    )
    if result.stderr:
        if "canonicalize" in result.stderr.lower():
            logger.debug(f"opengrep script detection skipped for {language} (canonicalize): {result.stderr[:200]}")
        else:
            logger.warning(f"opengrep stderr: {result.stderr[:500]}")

    # Parse JSON results - preserve raw opengrep structure
    findings = []
    if result.stdout:
        try:
            scan_output = json.loads(result.stdout)
            if "results" in scan_output:
                # Use raw opengrep findings to preserve full structure (start, end, extra, etc.)
                findings = scan_output["results"]
        except json.JSONDecodeError as e:
            logger.error(f"Failed to parse opengrep JSON output for {language}: {e}")

    return {"findings": findings, "files_scanned": files}


def scan_files_batch_optimized(
    files: List[str],
    config: Optional[str],
    scanner_config: ScannerConfig,
    base_path: str,
) -> Dict[str, Any]:
    """
    Scan a batch of files with OpenGrep (optimized for batching)

    Args:
        files: List of file paths to scan
        config: Optional scanner config
        scanner_config: scanner configuration object
        base_path: Base directory path

    Returns:
        Dict with 'findings' and 'files_scanned' keys
    """
    import subprocess

    if not files:
        return {"findings": [], "files_scanned": []}

    # Build OpenGrep command
    cmd = [_opengrep_exe(), "scan", "--json", "--quiet", "--no-git-ignore", "--scan-unknown-extensions"]

    # Add config
    if config:
        cmd.extend(["--config", config])
    else:
        # Detect ALL unique languages in this batch
        detected_languages = set()
        for file_path in files:
            language = scanner_config.detect_language_from_extension(file_path)
            if language:
                detected_languages.add(language)

        logger.info(f"Batch scan: Detected {len(detected_languages)} languages: {detected_languages}")

        # Get rules for all detected languages
        all_rules = set()
        for language in detected_languages:
            rules = scanner_config.get_security_rules_for_language(language)
            if rules:
                all_rules.update(rules)
                logger.info(f"  Language '{language}': loaded {len(rules)} rule paths")

        logger.info(f"Batch scan: Total {len(all_rules)} unique rule paths loaded")

        if all_rules:
            backend_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
            for rule in all_rules:
                # Registry paths (p/*, r/*) and URLs should be used as-is.
                # Local rule paths always end with .yaml — never treat those as
                # registry rules even if they start with "r/" (e.g. R language rules).
                is_registry = (
                    rule.startswith(("http://", "https://"))
                    or rule == "auto"
                    or (rule.startswith(("p/", "r/")) and not rule.endswith(".yaml"))
                )
                if is_registry:
                    cmd.extend(["--config", rule])
                elif not os.path.isabs(rule):
                    # Convert relative file paths to absolute
                    abs_rule = os.path.normpath(os.path.join(backend_dir, rule))
                    cmd.extend(["--config", abs_rule])
                else:
                    cmd.extend(["--config", rule])
            logger.info(f"opengrep command will use {len(all_rules)} config paths")
            # Log first few rule paths for debugging
            sample_rules = list(all_rules)[:3]
            logger.info(f"Sample rule paths: {sample_rules}")
        else:
            if getattr(sys, 'frozen', False):
                logger.warning(f"No rules found for detected languages in batch, skipping (frozen build)")
                return {"findings": [], "files_scanned": files}
            cmd.extend(["--config", "auto"])

    # Add files to scan
    cmd.extend(files)

    # Log command for debugging (truncate if too long)
    cmd_str = " ".join(cmd)
    if len(cmd_str) > 500:
        logger.info(f"Running opengrep command: {cmd_str[:500]}... (truncated)")
    else:
        logger.info(f"Running opengrep command: {cmd_str}")

    try:
        # Run opengrep with timeout (2 minutes per batch)
        result = subprocess.run(
            cmd,
            capture_output=True,
            text=True,
            timeout=120,
            cwd=base_path
        )

        # Log opengrep execution results
        stdout_len = len(result.stdout) if result.stdout else 0
        stderr_len = len(result.stderr) if result.stderr else 0
        logger.info(f"opengrep exit code: {result.returncode}, stdout length: {stdout_len}, stderr length: {stderr_len}")
        if result.stderr:
            logger.warning(f"opengrep stderr: {result.stderr[:500]}")

        # Parse results
        if result.stdout:
            scan_results = json.loads(result.stdout)
            findings = scan_results.get("results", [])

            # Ensure absolute paths
            for finding in findings:
                if "path" in finding:
                    finding["path"] = os.path.abspath(os.path.join(base_path, finding["path"]))

            return {
                "findings": findings,
                "files_scanned": files
            }

        return {"findings": [], "files_scanned": files}

    except subprocess.TimeoutExpired:
        logger.error(f"Batch scan timed out for {len(files)} files")
        return {"findings": [], "files_scanned": files, "error": "Batch timed out"}
    except Exception as e:
        logger.exception(f"Failed to scan batch of {len(files)} files")
        return {"findings": [], "files_scanned": files, "error": str(e)}


def scan_directory_optimized(
    target_path: str,
    config: Optional[str],
    scanner_config: ScannerConfig,
    selected_extensions: Optional[List[str]],
) -> Dict[str, Any]:
    """
    OPTIMIZED: Scan entire directory with a SINGLE OpenGrep invocation

    This is 10-20x faster than batching because OpenGrep only starts up once.

    Args:
        target_path: Directory path to scan
        config: Optional scanner config
        scanner_config: scanner configuration object
        selected_extensions: Optional list of file extensions to filter

    Returns:
        Dict with 'findings' and 'files_scanned' keys
    """
    import glob
    import subprocess

    # Get list of files to scan (for extension filtering and file counting)
    all_files = []
    if selected_extensions:
        for ext in selected_extensions:
            pattern = os.path.join(target_path, "**", f"*.{ext}")
            all_files.extend(glob.glob(pattern, recursive=True))
            # Also match pure dotfiles (e.g. .envrc when ext="envrc")
            all_files.extend(glob.glob(os.path.join(target_path, "**", f".{ext}"), recursive=True))
    else:
        # Detect languages in directory and get their extensions
        # Dynamically get all supported extensions from config
        detected_languages = set()
        all_extensions = list(scanner_config.extension_to_language_map.keys())
        for ext in all_extensions:
            pattern = f'*{ext}'
            found = (
                glob.glob(os.path.join(target_path, '**', pattern), recursive=True) or
                glob.glob(os.path.join(target_path, '**', ext), recursive=True)
            )
            if found:
                file_lang = scanner_config.detect_language_from_extension(ext)
                if file_lang:
                    detected_languages.add(file_lang)

        # Scan all files if no specific language detected
        pattern = os.path.join(target_path, "**", "*.*")
        all_files.extend(glob.glob(pattern, recursive=True))
        # Also match pure dotfiles (.env, .envrc, etc.) not captured by *.*
        for ext in scanner_config.extension_to_language_map.keys():
            all_files.extend(glob.glob(os.path.join(target_path, "**", ext), recursive=True))

    # Filter to files only
    all_files = [f for f in set(all_files) if os.path.isfile(f)]

    if not all_files:
        return {"findings": [], "files_scanned": []}

    # Build OpenGrep command - scan the ENTIRE directory at once
    cmd = [_opengrep_exe(), "scan", "--json", "--quiet", "--no-git-ignore", "--scan-unknown-extensions"]

    # Add config
    if config:
        cmd.extend(["--config", config])
    else:
        # Detect ALL unique languages in directory
        detected_languages = set()
        for file_path in all_files:
            language = scanner_config.detect_language_from_extension(file_path)
            if language:
                detected_languages.add(language)

        logger.info(f"Directory scan: Detected {len(detected_languages)} languages from {len(all_files)} files: {detected_languages}")

        # Get rules for all detected languages
        all_rules = set()
        for language in detected_languages:
            rules = scanner_config.get_security_rules_for_language(language)
            if rules:
                all_rules.update(rules)
                logger.info(f"  Language '{language}': loaded {len(rules)} rule paths")

        logger.info(f"Directory scan: Total {len(all_rules)} unique rule paths loaded")

        if all_rules:
            backend_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
            for rule in all_rules:
                # Registry paths (p/*, r/*) and URLs should be used as-is.
                # Local rule paths always end with .yaml — never treat those as
                # registry rules even if they start with "r/" (e.g. R language rules).
                is_registry = (
                    rule.startswith(("http://", "https://"))
                    or rule == "auto"
                    or (rule.startswith(("p/", "r/")) and not rule.endswith(".yaml"))
                )
                if is_registry:
                    cmd.extend(["--config", rule])
                elif not os.path.isabs(rule):
                    # Convert relative file paths to absolute
                    abs_rule = os.path.normpath(os.path.join(backend_dir, rule))
                    cmd.extend(["--config", abs_rule])
                else:
                    cmd.extend(["--config", rule])
            logger.info(f"opengrep command will use {len(all_rules)} config paths")
            logger.debug(f"First 3 rule paths: {list(all_rules)[:3]}")
        else:
            if getattr(sys, 'frozen', False):
                logger.warning(f"No rules found for detected languages in directory, skipping (frozen build)")
                return {"findings": [], "files_scanned": all_files}
            cmd.extend(["--config", "auto"])

    # Scan the directory (or all files if extension filtered)
    if selected_extensions:
        # If extensions specified, pass individual files
        cmd.extend(all_files)
    else:
        # Scan entire directory
        cmd.append(target_path)

    try:
        # Run opengrep with timeout (5 minutes max)
        result = subprocess.run(
            cmd,
            capture_output=True,
            text=True,
            timeout=code_analysis_settings.timeout_seconds,
            cwd=target_path
        )

        # Parse results
        if result.stdout:
            scan_results = json.loads(result.stdout)
            findings = scan_results.get("results", [])

            # Ensure absolute paths
            for finding in findings:
                if "path" in finding:
                    finding["path"] = os.path.abspath(os.path.join(target_path, finding["path"]))

            return {
                "findings": findings,
                "files_scanned": all_files
            }

        return {"findings": [], "files_scanned": all_files}

    except subprocess.TimeoutExpired:
        logger.error(f"Scan timed out after 5 minutes for directory: {target_path}")
        return {"findings": [], "files_scanned": [], "error": "Scan timed out after 5 minutes"}
    except Exception as e:
        logger.exception(f"Failed to scan directory: {target_path}")
        return {"findings": [], "files_scanned": [], "error": str(e)}


def scan_file_batch(
    files: List[str],
    config: Optional[str],
    scanner_config: ScannerConfig,
    base_path: str,
) -> List[Dict[str, Any]]:
    """
    DEPRECATED: Scan a batch of files with OpenGrep (kept for backwards compatibility)

    NOTE: This function is now deprecated in favor of scan_directory_optimized
    which is 10-20x faster by running OpenGrep once instead of multiple times.

    Args:
        files: List of file paths to scan
        config: Optional scanner config
        scanner_config: scanner configuration object
        base_path: Base directory path for relative path calculation

    Returns:
        List of findings from all files in the batch
    """
    if not files:
        return []

    # Build OpenGrep command
    cmd = [_opengrep_exe(), "scan", "--json", "--quiet", "--no-git-ignore", "--scan-unknown-extensions"]

    # Add config
    if config:
        cmd.extend(["--config", config])
    else:
        # Auto-detect language from first file
        language = scanner_config.detect_language_from_extension(files[0])
        if language:
            rules = scanner_config.get_security_rules_for_language(language)
            if rules:
                backend_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
                for rule in rules:
                    if not os.path.isabs(rule):
                        abs_rule = os.path.normpath(os.path.join(backend_dir, rule))
                        cmd.extend(["--config", abs_rule])
                    else:
                        cmd.extend(["--config", rule])
            else:
                if getattr(sys, 'frozen', False):
                    logger.warning(f"No rules for {language} in batch, skipping (frozen build)")
                    return []
                cmd.extend(["--config", "auto"])
        else:
            if getattr(sys, 'frozen', False):
                logger.warning(f"Could not detect language for batch, skipping (frozen build)")
                return []
            cmd.extend(["--config", "auto"])

    # Add files
    cmd.extend(files)

    try:
        result = subprocess.run(cmd, capture_output=True, text=True, cwd=base_path, timeout=600)

        # Parse results
        if result.stdout:
            scan_results = json.loads(result.stdout)
            findings = scan_results.get("results", [])

            # Ensure absolute paths in findings
            for finding in findings:
                if "path" in finding:
                    finding["path"] = os.path.abspath(os.path.join(base_path, finding["path"]))

            return findings
        return []
    except subprocess.TimeoutExpired:
        # Should not happen since we removed timeout
        logger.warning(f"Batch scan timed out for {len(files)} files")
        return []
    except Exception as e:
        logger.exception(f"Failed to scan batch of {len(files)} files")
        return []


class CodePatchRequest(BaseModel):
    """Request model for on-demand patch generation

    Attributes:
        file_path: Path to the file with the vulnerability
        line_number: Line number where the vulnerability was detected
        end_line: End line number for the vulnerability
        message: Scanner message describing the issue
        llm_config: Optional LLM configuration for generating AI recommendations
        fix_suggestion: Optional scanner autofix data (fix text or fixed_lines)
        original_line: Original code line(s) from scanner
    """
    file_path: str
    line_number: int
    end_line: Optional[int] = None
    message: str
    llm_config: Optional[Dict[str, Any]] = None
    fix_suggestion: Optional[str] = None
    fixed_lines: Optional[List[str]] = None
    original_line: Optional[str] = None


@router.post("/code/patch/generate")
async def generate_code_patch(
    request: CodePatchRequest,
    user: User = Depends(get_current_user),
):
    """
    Generate a @TODO-pq comment patch on-demand using LLM.

    Uses LLM to generate contextual, specific recommendations for each finding.
    Does NOT save to database - returns patch for immediate use.
    Patch will be saved to DB only when user applies it.

    Args:
        request: Patch generation request with file_path, line_number, message
        user: Authenticated user

    Returns:
        Dict with success status and generated patch_diff
    """
    # Rate limiting
    allowed, retry_after = patch_rate_limiter.is_allowed(user.id)
    if not allowed:
        raise HTTPException(
            status_code=429,
            detail=f"Rate limit exceeded. Retry after {retry_after} seconds",
            headers={"Retry-After": str(retry_after)},
        )
    
    try:
        # Validate line numbers
        if request.line_number < 1:
            raise ValueError("line_number must be >= 1")
        if request.end_line and request.end_line < request.line_number:
            raise ValueError("end_line must be >= line_number")
        if not request.message or not request.message.strip():
            raise ValueError("message is required")
        
        # Validate and normalize path
        file_path = validate_and_normalize_path(request.file_path)

        # Read file content
        with open(file_path, 'r', encoding='utf-8') as f:
            file_content = f.read()

        # Create a finding object for the unified prompt
        finding = {
            "start": {"line": request.line_number},
            "start_line": request.line_number,
            "message": request.message,
            "check_id": "scanner-finding",
            "extra": {
                "message": request.message,
                "severity": "WARNING"
            }
        }

        # Build prompt using unified prompt builder
        prompt = build_unified_fix_prompt(
            file_path=file_path,
            file_content=file_content,
            finding=finding,
            context_lines=10
        )

        # Strategy 1: Use LLM if configured
        llm_config = None
        if request.llm_config:
            llm_config = LLMConfig(
                provider=request.llm_config.get("provider", "google"),
                model=request.llm_config.get("model", "gemini/gemini-2.5-flash-lite"),
                api_key=request.llm_config.get("apiKey"),
                base_url=request.llm_config.get("baseUrl"),
            )

        patch_diff = None

        if llm_config and llm_config.api_key:
            # Use LLM to generate AI-based recommendation
            logger.info(f"Using LLM to generate patch for {file_path}:{request.line_number}")
            llm_response = generate_completion(prompt, llm_config)

            if llm_response:
                # Extract diff from LLM response (it should contain a unified diff)
                patch_diff = llm_response
                # Strip markdown code fences if present
                if "```" in patch_diff:
                    patch_diff = re.sub(r'^```[a-zA-Z]*\n|\n```$', '', patch_diff, flags=re.MULTILINE).strip()
                fence_start = patch_diff.find("--- ")
                if fence_start > -1:
                    patch_diff = patch_diff[fence_start:]

                # Validate that we got a valid diff
                if not patch_diff or "--- " not in patch_diff or "+++ " not in patch_diff:
                    logger.warning(f"LLM response didn't contain valid diff format")
                    patch_diff = None

        # Strategy 2: Fall back to scanner autofix if available
        if not patch_diff and request.fixed_lines:
            logger.info(f"Using scanner autofix for {file_path}:{request.line_number}")
            try:
                patch_diff = findings_to_patch(
                    file_path=file_path,
                    start_line=request.line_number,
                    end_line=request.end_line or request.line_number,
                    original_line=request.original_line,
                    fixed_lines=request.fixed_lines,
                )
                if not patch_diff:
                    logger.warning(f"scanner autofix failed to generate patch")
            except Exception as e:
                logger.error(f"Error generating scanner autofix patch: {e}")
                patch_diff = None

        # Strategy 3: Fall back to simple comment
        if not patch_diff:
            logger.info(f"Falling back to comment patch for {file_path}:{request.line_number}")
            patch_diff = generate_comment_patch(
                file_path=file_path,
                line_number=request.line_number,
                message=request.message
            )

        if not patch_diff:
            raise HTTPException(
                status_code=500,
                detail=f"Failed to generate patch for {file_path}:{request.line_number}"
            )

        logger.info(f"Generated LLM-based patch for {file_path}:{request.line_number}")

        return {
            "success": True,
            "patch_diff": patch_diff,
            "file_path": file_path,
            "line_number": request.line_number,
        }

    except ValueError as e:
        logger.error(f"Validation error in generate-patch: {str(e)}")
        raise HTTPException(status_code=400, detail=str(e))
    except HTTPException:
        raise
    except Exception as e:
        logger.exception(f"Failed to generate scanner patch")
        safe_msg = sanitize_error_message(e, "Patch generation")
        raise HTTPException(status_code=500, detail=f"Patch generation failed: {safe_msg}")

