class EnterpriseChunkingPipeline:
    """Enterprise-grade chunking pipeline with quality assessment."""

    def __init__(self, config: Dict[str, Any]):
        self.config = config
        self.chunker = HierarchicalChunker(
            max_chunk_size=config.get("max_chunk_size", 1000),
            overlap_ratio=config.get("overlap_ratio", 0.1)
        )
        self.quality_assessor = ChunkQualityAssessor()
        self.strategy_selector = ProcessingStrategySelector()

        # Default configuration with enterprise settings
        self.default_config = {
            "max_chunk_size": 1000,
            "overlap_ratio": 0.15,
            "min_quality_threshold": 0.7,
            "enable_optimization": True,
            "processing_timeout": 300,  # 5 minutes
            "parallel_processing": True,
            "cache_results": True
        }

        # Merge with provided config
        self.config = {**self.default_config, **config}
        self.performance_monitor = ProcessingPerformanceMonitor()

    def process_document_with_quality_control(self, document: Document) -> Dict[str, Any]:
        """Process document with comprehensive quality assessment."""
        start_time = time.time()

        try:
            # Step 1: Analyze document characteristics
            doc_analysis = self._analyze_document_complexity(document)

            # Step 2: Select optimal processing strategy
            strategy = self.strategy_selector.select_strategy(doc_analysis)

            # Step 3: Create initial chunks using selected strategy
            chunks = self._process_with_strategy(document, strategy)

            # Step 4: Assess chunk quality
            quality_metrics = self.quality_assessor.assess_chunk_quality(chunks)

            # Step 5: Apply quality-based optimization if needed
            if (quality_metrics["overall_quality"] <
                self.config.get("min_quality_threshold", 0.7)):

                optimized_chunks = self._optimize_chunks(chunks, quality_metrics)
                final_quality = self.quality_assessor.assess_chunk_quality(optimized_chunks)
                chunks = optimized_chunks
                quality_metrics = final_quality

            # Step 6: Record performance metrics
            processing_time = time.time() - start_time
            self.performance_monitor.record_processing(
                doc_length=len(document.page_content),
                chunk_count=len(chunks),
                quality_score=quality_metrics["overall_quality"],
                processing_time=processing_time,
                strategy_used=strategy
            )

        except Exception as e:
            self._handle_processing_error(e, document)
            return {"error": str(e), "chunks": [], "quality_metrics": {}}

        return {
            "chunks": chunks,
            "document_analysis": doc_analysis,
            "quality_metrics": quality_metrics,
            "processing_stats": {
                "original_length": len(document.page_content),
                "chunk_count": len(chunks),
                "avg_chunk_size": sum(len(c.page_content) for c in chunks) / len(chunks),
                "quality_score": quality_metrics["overall_quality"],
                "processing_time": processing_time,
                "strategy_used": strategy
            }
        }

def process_advanced_table(table_text: str) -> Dict[str, Any]:
    """Process table content with advanced structure preservation."""
    lines = table_text.strip().split('\n')
    table_lines = [line for line in lines if '|' in line and len(line.split('|')) > 2]

    if not table_lines:
        return {"error": "No valid table structure found"}

    # Extract and analyze table structure
    header_row = table_lines[0]
    headers = [cell.strip() for cell in header_row.split('|') if cell.strip()]

    # Process data rows
    data_rows = []
    for line in table_lines[1:]:
        if not line.strip().startswith('|'):  # Skip separator lines
            continue
        row_data = [cell.strip() for cell in line.split('|') if cell.strip()]
        if len(row_data) == len(headers):
            data_rows.append(row_data)

    # Analyze table content
    numeric_columns = []
    for i, header in enumerate(headers):
        column_values = [row[i] for row in data_rows if i < len(row)]
        if any(re.match(r'^[\d,.$%]+$', val.replace(' ', '')) for val in column_values):
            numeric_columns.append(header)

    # Generate rich description
    description_parts = []
    description_parts.append(f"Table with {len(data_rows)} rows and {len(headers)} columns")

    if headers:
        description_parts.append(f"Headers: {', '.join(headers)}")

    if numeric_columns:
        description_parts.append(f"Numeric data in columns: {', '.join(numeric_columns)}")

    # Identify key patterns
    if any('total' in header.lower() for header in headers):
        description_parts.append("Contains summary/total information")

    if any('date' in header.lower() or 'time' in header.lower() for header in headers):
        description_parts.append("Contains temporal data")

    enhanced_description = ". ".join(description_parts) + "."

    return {
        "enhanced_content": f"{enhanced_description}\n\n{table_text}",
        "metadata": {
            "content_type": "table",
            "row_count": len(data_rows),
            "column_count": len(headers),
            "headers": headers,
            "numeric_columns": numeric_columns,
            "data_types": self._analyze_column_types(data_rows, headers),
            "table_complexity": "high" if len(data_rows) > 10 or len(headers) > 5 else "medium"
        }
    }

def process_code_block(code_text: str, language: str = None) -> Dict[str, Any]:
    """Process code blocks with syntax awareness."""
    lines = code_text.split('\n')

    # Detect programming language if not specified
    if not language:
        language = self._detect_programming_language(code_text)

    # Analyze code structure
    analysis = {
        "language": language,
        "line_count": len(lines),
        "complexity": self._assess_code_complexity(code_text, language),
        "contains_functions": bool(re.search(r'\bdef\b|\bfunction\b|\bclass\b', code_text)),
        "contains_imports": bool(re.search(r'\bimport\b|\brequire\b|\binclude\b', code_text)),
        "contains_comments": bool(re.search(r'#.*|//.*|/\*.*\*/', code_text))
    }

def process_image_reference(image_ref: str, context: str = "") -> Dict[str, Any]:
    """Process image references with contextual description."""
    # Extract image information
    image_info = {
        "type": "image",
        "filename": os.path.basename(image_ref),
        "extension": os.path.splitext(image_ref)[1].lower(),
        "context": context
    }

    # Generate descriptive text based on filename and context
    filename_base = os.path.splitext(image_info["filename"])[0]
    description_parts = []

    # Infer content from filename patterns
    if any(word in filename_base.lower() for word in ['chart', 'graph', 'plot']):
        description_parts.append("Data visualization showing")
    elif any(word in filename_base.lower() for word in ['diagram', 'architecture', 'flow']):
        description_parts.append("Technical diagram illustrating")
    elif 'screenshot' in filename_base.lower():
        description_parts.append("Screenshot demonstrating")
    else:
        description_parts.append("Image depicting")

    # Add context if available
    if context:
        description_parts.append(f"related to {context}")

    # Create searchable description
    searchable_description = f"[Image: {' '.join(description_parts)} - {image_info['filename']}]"

    return {
        "enhanced_content": searchable_description,
        "metadata": {
            **image_info,
            "searchable_description": searchable_description
        }
    }

class AdvancedProcessingPipeline:
    """Complete advanced document processing pipeline."""

    def __init__(self, max_chunk_size: int = 1000, enable_quality_assessment: bool = True):
        self.max_chunk_size = max_chunk_size
        self.enable_quality_assessment = enable_quality_assessment

        # Initialize processors
        self.metadata_chunker = MetadataEnhancedChunker(max_chunk_size=max_chunk_size)
        self.quality_assessor = ChunkQualityAssessor() if enable_quality_assessment else None
        self.table_processor = TableProcessor()
        self.code_processor = CodeBlockProcessor()
        self.image_processor = ImageProcessor()

    def process_document(self, document: Document) -> Dict[str, Any]:
        """Process document using the most appropriate strategy."""

        # Step 1: Analyze document characteristics
        doc_analysis = self._analyze_document_complexity(document)

        # Step 2: Choose processing strategy based on content
        if doc_analysis["has_tables"]:
            print("Detected tables - using table-aware processing...")
            processed_chunks = self._process_with_table_awareness(document)
        elif doc_analysis["has_code"]:
            print("Detected code blocks - using code-aware processing...")
            processed_chunks = self._process_with_code_awareness(document)
        else:
            print("Using standard hierarchical processing...")
            processed_chunks = self.metadata_chunker.create_enhanced_chunks(document)

    def _process_with_table_awareness(self, document: Document) -> List[Document]:
        """Process document with special table handling."""
        content = document.page_content
        chunks = []

        # Split content into sections, preserving table boundaries
        sections = self._split_preserving_tables(content)

        for section in sections:
            if self._contains_table(section):
                # Special table processing
                table_chunk = self._create_table_chunk(section, document.metadata)
                chunks.append(table_chunk)
            else:
                # Standard hierarchical processing for non-table sections
                section_doc = Document(page_content=section, metadata=document.metadata)
                section_chunks = self.metadata_chunker.create_enhanced_chunks(section_doc)
                chunks.extend(section_chunks)

        return chunks

    def _process_with_code_awareness(self, document: Document) -> List[Document]:
        """Process document with special code block handling."""
        content = document.page_content
        chunks = []

        # Identify and preserve code block boundaries
        code_sections = self._identify_code_sections(content)

        for section_type, section_content in code_sections:
            if section_type == "code":
                # Special code processing
                code_chunk = self._create_code_chunk(section_content, document.metadata)
                chunks.append(code_chunk)
            else:
                # Standard processing for non-code sections
                section_doc = Document(page_content=section_content, metadata=document.metadata)
                section_chunks = self.metadata_chunker.create_enhanced_chunks(section_doc)
                chunks.extend(section_chunks)

        return chunks

        # Step 3: Assess quality if enabled
        quality_metrics = {}
        if self.enable_quality_assessment and self.quality_assessor:
            quality_metrics = self.quality_assessor.assess_chunk_quality(processed_chunks)

            # Apply optimization if quality is below threshold
            if quality_metrics.get("overall_quality", 0) < 0.7:
                processed_chunks = self._optimize_low_quality_chunks(processed_chunks, quality_metrics)
                quality_metrics = self.quality_assessor.assess_chunk_quality(processed_chunks)

        # Step 4: Add processing metadata to all chunks
        for chunk in processed_chunks:
            chunk.metadata.update({
                "processing_strategy": doc_analysis["recommended_strategy"],
                "document_complexity": doc_analysis["complexity_score"],
                "processing_pipeline": "advanced_v2",
                "processed_at": datetime.now().isoformat()
            })

        # Step 5: Compile comprehensive results
        return {
            "chunks": processed_chunks,
            "document_analysis": doc_analysis,
            "quality_metrics": quality_metrics,
            "processing_stats": {
                "chunk_count": len(processed_chunks),
                "total_processed_chars": sum(len(c.page_content) for c in processed_chunks),
                "avg_chunk_size": sum(len(c.page_content) for c in processed_chunks) / len(processed_chunks),
                "content_types_detected": doc_analysis.get("content_types", []),
                "processing_time": time.time() - start_time
            }
        }

    def _analyze_document_complexity(self, document: Document) -> Dict[str, Any]:
        """Analyze document to determine optimal processing strategy."""
        content = document.page_content

        # Detect various content types with enhanced patterns
        has_tables = self._detect_tables(content)
        has_code = self._detect_code_blocks(content)
        has_lists = self._detect_lists(content)
        has_headings = self._detect_headings(content)
        has_images = self._detect_images(content)
        has_equations = self._detect_equations(content)

        # Analyze document structure depth
        structure_analysis = self._analyze_document_structure(content)

    def _detect_tables(self, content: str) -> bool:
        """Advanced table detection."""
        table_patterns = [
            r'\|.*\|.*\|',  # Pipe-separated tables
            r'┌.*┬.*┐',     # ASCII box tables
            r'\+[-=]+\+',   # Grid tables
        ]

        for pattern in table_patterns:
            if len(re.findall(pattern, content)) >= 2:
                return True
        return False

    def _detect_code_blocks(self, content: str) -> bool:
        """Enhanced code block detection."""
        code_indicators = [
            r'```[\w]*\n.*?```',  # Fenced code blocks
            r'^    \w+.*',        # Indented code (4 spaces)
            r'^\t\w+.*',          # Tab-indented code
            r'<code>.*?</code>',  # HTML code tags
        ]

        code_lines = 0
        for pattern in code_indicators:
            matches = re.findall(pattern, content, re.MULTILINE | re.DOTALL)
            code_lines += sum(len(match.split('\n')) for match in matches)

        return code_lines > 5  # Significant code presence

    def _detect_equations(self, content: str) -> bool:
        """Detect mathematical equations."""
        equation_patterns = [
            r'\$\$.*?\$\$',     # LaTeX display math
            r'\$.*?\$',         # LaTeX inline math
            r'\\begin\{.*?\}',  # LaTeX environments
        ]

        for pattern in equation_patterns:
            if re.search(pattern, content, re.DOTALL):
                return True
        return False

        # Calculate weighted complexity score
        complexity_score = 0
        content_types = []

        if has_tables:
            complexity_score += 4
            content_types.append("tables")
        if has_code:
            complexity_score += 3
            content_types.append("code")
        if has_equations:
            complexity_score += 3
            content_types.append("equations")
        if has_images:
            complexity_score += 2
            content_types.append("images")
        if has_lists:
            complexity_score += 1
            content_types.append("lists")
        if has_headings:
            complexity_score += 2
            content_types.append("headings")

        # Add structural complexity
        structure_complexity = structure_analysis.get("depth_score", 0)
        complexity_score += structure_complexity

        # Determine optimal processing strategy
        if has_tables and has_code:
            strategy = "multi_modal_advanced"
        elif has_tables:
            strategy = "table_aware"
        elif has_code:
            strategy = "code_aware"
        elif complexity_score > 6:
            strategy = "hierarchical_advanced"
        elif complexity_score > 3:
            strategy = "hierarchical_standard"
        else:
            strategy = "simple"

        return {
            "has_tables": has_tables,
            "has_code": has_code,
            "has_lists": has_lists,
            "has_headings": has_headings,
            "has_images": has_images,
            "has_equations": has_equations,
            "complexity_score": complexity_score,
            "content_types": content_types,
            "structure_analysis": structure_analysis,
            "recommended_strategy": strategy
        }

    def _optimize_low_quality_chunks(self, chunks: List[Document],
                                   quality_metrics: Dict[str, float]) -> List[Document]:
        """Automatically optimize chunks with low quality scores."""
        optimized_chunks = []

        for chunk in chunks:
            chunk_quality = self._assess_individual_chunk_quality(chunk)

            if chunk_quality["overall"] < 0.6:
                # Apply optimization strategies
                if chunk_quality["coherence"] < 0.5:
                    # Merge with adjacent chunks for better coherence
                    optimized_chunk = self._merge_for_coherence(chunk, chunks)
                elif chunk_quality["density"] < 0.4:
                    # Split overly dense chunks
                    split_chunks = self._split_dense_chunk(chunk)
                    optimized_chunks.extend(split_chunks)
                    continue
                elif chunk_quality["completeness"] < 0.5:
                    # Enhance with additional context
                    optimized_chunk = self._enhance_with_context(chunk)
                else:
                    optimized_chunk = chunk

                optimized_chunks.append(optimized_chunk)
            else:
                optimized_chunks.append(chunk)

        return optimized_chunks

class ProcessingPerformanceMonitor:
    """Monitor and analyze processing performance."""

    def __init__(self):
        self.processing_history = []
        self.performance_metrics = {
            "avg_processing_time": 0,
            "avg_quality_score": 0,
            "strategy_effectiveness": {},
            "error_rates": {}
        }

    def record_processing(self, doc_length: int, chunk_count: int,
                         quality_score: float, processing_time: float,
                         strategy_used: str):
        """Record processing metrics for analysis."""
        record = {
            "timestamp": datetime.now(),
            "doc_length": doc_length,
            "chunk_count": chunk_count,
            "quality_score": quality_score,
            "processing_time": processing_time,
            "strategy_used": strategy_used,
            "efficiency": quality_score / processing_time  # Quality per second
        }

        self.processing_history.append(record)
        self._update_performance_metrics()

    def get_performance_report(self) -> Dict[str, Any]:
        """Generate comprehensive performance report."""
        recent_records = self.processing_history[-100:]  # Last 100 processings

        if not recent_records:
            return {"error": "No processing history available"}

        return {
            "total_documents_processed": len(self.processing_history),
            "recent_avg_quality": np.mean([r["quality_score"] for r in recent_records]),
            "recent_avg_processing_time": np.mean([r["processing_time"] for r in recent_records]),
            "most_effective_strategy": self._get_most_effective_strategy(),
            "quality_trend": self._analyze_quality_trend(),
            "recommendations": self._generate_optimization_recommendations()
        }

# Complex test document
complex_document = """
# System Architecture Guide

This guide covers our microservices architecture implementation.

## Database Schema

The following table shows our main entities:

| Entity | Fields | Relationships |
|--------|--------|---------------|
| User | id, name, email | HasMany Orders |
| Order | id, user_id, total | BelongsTo User |
| Product | id, name, price | BelongsToMany Orders |

## Implementation Example

Here's the core service implementation:

```python
class OrderService:
    def __init__(self, db_connection):
        self.db = db_connection

    def create_order(self, user_id, items):
        order = Order(user_id=user_id)
        for item in items:
            order.add_item(item)
        return order.save()

# Set up performance monitoring
monitor = ProcessingPerformanceMonitor()
pipeline = AdvancedProcessingPipeline()

# Process multiple documents with monitoring
test_documents = [
    "Simple text document with basic content.",
    complex_document,  # From Exercise 1
    "Document with tables: | A | B |\n|---|---|\n| 1 | 2 |",
    "Code document:\n```python\nprint('hello')\n```"
]

for i, doc_content in enumerate(test_documents):
    doc = Document(page_content=doc_content)
    start_time = time.time()

    result = pipeline.process_document(doc)
    processing_time = time.time() - start_time

    monitor.record_processing(
        doc_length=len(doc_content),
        chunk_count=len(result['chunks']),
        quality_score=result['quality_metrics'].get('overall_quality', 0),
        processing_time=processing_time,
        strategy_used=result['document_analysis']['recommended_strategy']
    )

# Generate performance report
report = monitor.get_performance_report()
print("\nPerformance Report:")
print(f"Documents processed: {report['total_documents_processed']}")
print(f"Average quality: {report['recent_avg_quality']:.2f}")
print(f"Average processing time: {report['recent_avg_processing_time']:.3f}s")
print(f"Most effective strategy: {report['most_effective_strategy']}")

class ScalableProcessingPipeline(AdvancedProcessingPipeline):
    """Production-ready pipeline with scaling capabilities."""

    def __init__(self, config):
        super().__init__(**config)
        self.parallel_workers = config.get('parallel_workers', 4)
        self.batch_size = config.get('batch_size', 10)
        self.cache = ProcessingCache()
        self.rate_limiter = RateLimiter()

    def process_documents_batch(self, documents: List[Document]) -> List[Dict]:
        """Process multiple documents in parallel."""
        from concurrent.futures import ThreadPoolExecutor, as_completed

        results = []
        with ThreadPoolExecutor(max_workers=self.parallel_workers) as executor:
            # Submit all documents for processing
            future_to_doc = {
                executor.submit(self.process_document, doc): doc
                for doc in documents
            }

            # Collect results as they complete
            for future in as_completed(future_to_doc):
                doc = future_to_doc[future]
                try:
                    result = future.result()
                    results.append(result)
                except Exception as e:
                    results.append({"error": str(e), "document": doc})

        return results

    def _handle_processing_error(self, error: Exception, document: Document) -> Dict:
        """Handle processing errors gracefully."""
        error_info = {
            "error_type": type(error).__name__,
            "error_message": str(error),
            "document_length": len(document.page_content),
            "timestamp": datetime.now().isoformat()
        }

        # Log error for monitoring
        self.error_logger.log_error(error_info)

        # Attempt fallback processing
        try:
            fallback_chunks = self._fallback_processing(document)
            return {
                "chunks": fallback_chunks,
                "quality_metrics": {"overall_quality": 0.3},  # Low quality indicator
                "processing_stats": {"fallback_used": True},
                "error_info": error_info
            }
        except Exception as fallback_error:
            return {
                "chunks": [],
                "error": f"Primary: {error}. Fallback: {fallback_error}",
                "processing_stats": {"total_failure": True}
            }

class ProductionMonitoring:
    """Production monitoring and alerting system."""

    def __init__(self):
        self.alerts = []
        self.metrics_collector = MetricsCollector()

    def check_system_health(self, processing_results: List[Dict]) -> Dict[str, Any]:
        """Check overall system health and generate alerts."""
        health_metrics = {
            "success_rate": self._calculate_success_rate(processing_results),
            "avg_quality": self._calculate_avg_quality(processing_results),
            "processing_time_p95": self._calculate_percentile(processing_results, 'processing_time', 95),
            "error_types": self._analyze_error_types(processing_results)
        }

        # Generate alerts for anomalies
        alerts = []
        if health_metrics["success_rate"] < 0.95:
            alerts.append("LOW_SUCCESS_RATE")
        if health_metrics["avg_quality"] < 0.6:
            alerts.append("LOW_QUALITY_SCORES")
        if health_metrics["processing_time_p95"] > 30:  # 30 seconds
            alerts.append("HIGH_PROCESSING_LATENCY")

        return {
            "health_metrics": health_metrics,
            "alerts": alerts,
            "system_status": "HEALTHY" if not alerts else "DEGRADED"
        }