⚙️ Session 4: Advanced Prompt Engineering¶

class RAGPromptEngineer:
    """Advanced prompt engineering specifically for RAG systems."""

    def __init__(self, llm_model):
        self.llm_model = llm_model

        # Domain-specific prompt templates
        self.prompt_templates = {
            'factual_qa': self._factual_qa_template,
            'analytical': self._analytical_template,
            'procedural': self._procedural_template,
            'comparative': self._comparative_template,
            'creative': self._creative_template,
            'technical': self._technical_template
        }

        # Prompt optimization techniques
        self.optimization_techniques = {
            'chain_of_thought': self._add_chain_of_thought,
            'source_weighting': self._add_source_weighting,
            'confidence_calibration': self._add_confidence_calibration,
            'multi_step_reasoning': self._add_multi_step_reasoning,
            'context_analysis': self._add_context_analysis,
            'uncertainty_handling': self._add_uncertainty_handling
        }

def engineer_rag_prompt(self, query: str, context: str,
                       query_type: str = 'factual_qa',
                       optimizations: List[str] = ['confidence_calibration']) -> Dict[str, Any]:
    """Engineer optimized RAG prompt with specified techniques."""

    # Start with base template
    base_prompt = self.prompt_templates[query_type](query, context)

    # Apply optimization techniques
    optimized_prompt = base_prompt
    optimization_metadata = {}

    for technique in optimizations:
        if technique in self.optimization_techniques:
            result = self.optimization_techniques[technique](
                optimized_prompt, query, context
            )
            optimized_prompt = result['prompt']
            optimization_metadata[technique] = result['metadata']

    return {
        'base_prompt': base_prompt,
        'optimized_prompt': optimized_prompt,
        'optimizations_applied': optimizations,
        'optimization_metadata': optimization_metadata,
        'query_type': query_type,
        'estimated_tokens': len(optimized_prompt.split()),
        'complexity_score': self._calculate_prompt_complexity(optimized_prompt)
    }

def _factual_qa_template(self, query: str, context: str) -> str:
    """Optimized template for factual question answering."""

    return f"""You are an expert information analyst. Use the provided context to answer the user's question accurately and comprehensively.

CONTEXT INFORMATION:
{context}

QUESTION: {query}

INSTRUCTIONS:
1. Base your answer strictly on the provided context
2. If the context doesn't contain sufficient information, clearly state this limitation
3. Cite specific sources when making claims using [Source: X] format
4. Provide direct quotes when appropriate using "quote text" format
5. Distinguish between facts and interpretations
6. If there are contradictions in the sources, acknowledge and explain them
7. Structure your answer logically with clear supporting evidence

ANSWER STRUCTURE:
- Start with a direct answer to the question
- Provide supporting details from the context with citations
- Include relevant quotes with source attribution
- Note any limitations or uncertainties
- End with a summary if the answer is complex

ANSWER:"""

def _analytical_template(self, query: str, context: str) -> str:
    """Template for analytical queries requiring synthesis."""

    return f"""You are an expert analyst. Analyze the provided context to answer the analytical question comprehensively.

CONTEXT FOR ANALYSIS:
{context}

ANALYTICAL QUESTION: {query}

ANALYSIS REQUIREMENTS:
1. Examine multiple perspectives presented in the context
2. Identify patterns, trends, or relationships
3. Compare and contrast different viewpoints or approaches
4. Draw evidence-based conclusions
5. Acknowledge limitations in the available information
6. Provide balanced assessment considering all evidence

ANALYSIS STRUCTURE:
1. OVERVIEW: Brief summary of the analytical question and available evidence
2. KEY FINDINGS: Main insights derived from the context
3. COMPARATIVE ANALYSIS: Different perspectives or approaches identified
4. SUPPORTING EVIDENCE: Specific examples and citations from context
5. LIMITATIONS: What the analysis cannot determine from available context
6. CONCLUSION: Synthesized answer based on comprehensive analysis

ANALYSIS:"""

def _technical_template(self, query: str, context: str) -> str:
    """Template for technical implementation questions."""

    return f"""You are a technical expert. Provide accurate technical guidance based on the provided documentation and context.

TECHNICAL CONTEXT:
{context}

TECHNICAL QUESTION: {query}

TECHNICAL RESPONSE REQUIREMENTS:
1. Provide accurate, implementable guidance
2. Include specific technical details, parameters, and configurations
3. Reference official documentation when available
4. Explain prerequisites and dependencies
5. Mention common pitfalls and troubleshooting tips
6. Provide code examples if present in context
7. Clarify any assumptions or limitations

RESPONSE STRUCTURE:
- DIRECT ANSWER: Clear solution to the technical question
- IMPLEMENTATION DETAILS: Step-by-step guidance with specifics
- PREREQUISITES: Required setup, tools, or dependencies
- CODE EXAMPLES: Relevant code snippets from context (if any)
- TROUBLESHOOTING: Common issues and solutions
- ADDITIONAL CONSIDERATIONS: Performance, security, or best practices

TECHNICAL RESPONSE:"""

def _add_chain_of_thought(self, base_prompt: str, query: str,
                        context: str) -> Dict[str, Any]:
    """Add chain-of-thought reasoning to prompt."""

    cot_enhancement = """

REASONING PROCESS:
Before providing your final answer, work through your reasoning step-by-step:

STEP 1 - CONTEXT ANALYSIS:
- What key information from the context relates to the question?
- How reliable and comprehensive is this information?
- Are there any gaps or contradictions in the context?

STEP 2 - INFORMATION EXTRACTION:
- What are the most relevant facts, data points, or insights?
- Which sources provide the strongest evidence?
- What examples or details support the main points?

STEP 3 - LOGICAL SYNTHESIS:
- How do the different pieces of information connect?
- What patterns or relationships can be identified?
- How do you resolve any contradictions or ambiguities?

STEP 4 - ANSWER CONSTRUCTION:
- What is the most accurate and complete answer based on the evidence?
- What are the confidence levels for different parts of the answer?
- What limitations or uncertainties should be acknowledged?

Let me work through this systematically:

STEP 1 - Context Analysis:
[Analyze the quality and relevance of provided context]

STEP 2 - Information Extraction:
[Extract and organize key information]

STEP 3 - Logical Synthesis:
[Connect information and identify patterns]

STEP 4 - Answer Construction:
[Build comprehensive, evidence-based answer]

FINAL ANSWER:
"""

    enhanced_prompt = base_prompt + cot_enhancement

    return {
        'prompt': enhanced_prompt,
        'metadata': {
            'technique': 'chain_of_thought',
            'added_tokens': len(cot_enhancement.split()),
            'reasoning_steps': 4,
            'systematic_analysis': True
        }
    }

def _add_confidence_calibration(self, base_prompt: str, query: str,
                              context: str) -> Dict[str, Any]:
    """Add advanced confidence calibration framework."""

    confidence_framework = self._build_advanced_confidence_framework()
    enhanced_prompt = base_prompt + confidence_framework

    return {
        'prompt': enhanced_prompt,
        'metadata': {
            'technique': 'confidence_calibration',
            'added_tokens': len(confidence_framework.split()),
            'assessment_dimensions': 5,
            'calibration_levels': ['evidence', 'completeness', 'certainty', 'limitations', 'overall']
        }
    }

def _build_advanced_confidence_framework(self) -> str:
    """Build comprehensive confidence assessment framework."""

    return """

CONFIDENCE ASSESSMENT FRAMEWORK:
Provide a detailed confidence assessment for your response:

1. EVIDENCE STRENGTH (Strong/Medium/Weak):
   - Strong: Multiple reliable, consistent sources with detailed information
   - Medium: Some sources with mostly consistent information, minor gaps
   - Weak: Limited sources, significant gaps, or inconsistent information

2. COMPLETENESS (Complete/Substantial/Partial/Limited):
   - Complete: Context fully addresses all aspects of the question
   - Substantial: Context addresses most important aspects with minor gaps
   - Partial: Context addresses some aspects but missing key information
   - Limited: Context has minimal relevant information for the question

3. CERTAINTY LEVEL (High/Moderate/Low):
   - High: Clear, unambiguous information with strong supporting evidence
   - Moderate: Generally clear information with some ambiguities or uncertainties
   - Low: Ambiguous or conflicting information requiring interpretation

4. KEY LIMITATIONS:
   - Information gaps: What important information is missing?
   - Source limitations: Are sources outdated, biased, or incomplete?
   - Scope limitations: What aspects of the question cannot be addressed?
   - Interpretation uncertainties: What requires subjective judgment?

5. OVERALL CONFIDENCE SCORE (0-100%):
   - 90-100%: Highly confident, comprehensive evidence base, minimal limitations
   - 70-89%: Confident, adequate evidence base, minor limitations acknowledged
   - 50-69%: Moderately confident, some evidence gaps or uncertainties
   - 30-49%: Low confidence, significant limitations or contradictions
   - Below 30%: Very low confidence, insufficient or unreliable evidence

CONFIDENCE ASSESSMENT:
- Evidence Strength: [Strong/Medium/Weak] - [Brief explanation]
- Completeness: [Complete/Substantial/Partial/Limited] - [Brief explanation]
- Certainty Level: [High/Moderate/Low] - [Brief explanation]
- Key Limitations: [List 2-3 most significant limitations]
- Overall Confidence Score: [0-100%] - [Justification for score]

RELIABILITY INDICATORS:
- Source diversity: [Number of different sources consulted]
- Information recency: [How current is the information]
- Cross-validation: [Are claims supported by multiple sources]
"""

def _add_context_analysis(self, base_prompt: str, query: str,
                         context: str) -> Dict[str, Any]:
    """Add context quality analysis to improve response accuracy."""

    context_analysis_enhancement = """

CONTEXT QUALITY ANALYSIS:
Before answering, analyze the quality and characteristics of the provided context:

CONTEXT ASSESSMENT:
1. RELEVANCE ANALYSIS:
   - How directly does the context address the specific question asked?
   - What percentage of the context is relevant vs. tangential?
   - Are there any irrelevant sections that should be ignored?

2. SOURCE EVALUATION:
   - How many distinct sources are represented in the context?
   - What types of sources are these (official docs, blogs, academic, etc.)?
   - Do sources appear credible and authoritative for this topic?

3. INFORMATION QUALITY:
   - Is the information current and up-to-date?
   - Are there any obvious errors, inconsistencies, or contradictions?
   - How detailed and comprehensive is the coverage?

4. COVERAGE ANALYSIS:
   - What aspects of the question are well-covered by the context?
   - What important aspects are missing or under-represented?
   - Are there gaps that affect the completeness of possible answers?

Based on this context analysis, I will now provide the most accurate answer possible while clearly acknowledging any limitations imposed by context quality.

CONTEXT ANALYSIS SUMMARY:
[Brief assessment of context quality and relevance]

"""

def _add_multi_step_reasoning(self, base_prompt: str, query: str,
                            context: str) -> Dict[str, Any]:
    """Add multi-step reasoning for complex analytical queries."""

    multi_step_enhancement = """

MULTI-STEP REASONING PROCESS:
For complex questions requiring synthesis, use this structured approach:

STEP 1 - QUESTION DECOMPOSITION:
- Break down the main question into smaller, answerable components
- Identify what types of information are needed for each component
- Determine the logical relationships between different components

STEP 2 - INFORMATION MAPPING:
- Map relevant context sections to each question component
- Identify which sources address which aspects of the question
- Note any information gaps or conflicts between sources

STEP 3 - COMPONENT ANALYSIS:
- Analyze each question component using relevant context
- Draw preliminary conclusions for each component
- Assess confidence level for each component answer

STEP 4 - INTEGRATION AND SYNTHESIS:
- Combine component analyses into a coherent overall answer
- Resolve any conflicts or contradictions between components
- Identify emergent insights from the integrated analysis

STEP 5 - VALIDATION AND REFINEMENT:
- Check if the integrated answer fully addresses the original question
- Verify that conclusions are supported by evidence
- Refine the answer to improve clarity and accuracy

MULTI-STEP ANALYSIS:

STEP 1 - Question Decomposition:
[Break down the question into components]

STEP 2 - Information Mapping:
[Map context to question components]

STEP 3 - Component Analysis:
[Analyze each component separately]

STEP 4 - Integration and Synthesis:
[Combine analyses coherently]

STEP 5 - Validation and Refinement:
[Final validation and refinement]

INTEGRATED FINAL ANSWER:
"""

class DynamicPromptAdapter:
    """Adapt prompts based on context quality and query characteristics."""

    def __init__(self, llm_model):
        self.llm_model = llm_model
        self.prompt_engineer = RAGPromptEngineer(llm_model)

    def adapt_prompt_dynamically(self, query: str, context: str,
                                retrieval_metadata: Dict) -> Dict[str, Any]:
        """Dynamically adapt prompt based on context and query analysis."""

        # Step 1: Analyze input characteristics
        context_analysis = self._analyze_context_quality(context, query)
        query_analysis = self._analyze_query_characteristics(query)

        # Step 2: Select optimal strategy
        prompt_strategy = self._select_prompt_strategy(
            context_analysis, query_analysis, retrieval_metadata
        )

        # Step 3: Generate adapted prompt
        adapted_prompt = self._generate_adapted_prompt(
            query, context, prompt_strategy
        )

        return {
            'adapted_prompt': adapted_prompt,
            'context_analysis': context_analysis,
            'query_analysis': query_analysis,
            'strategy': prompt_strategy,
            'adaptation_reasoning': self._explain_adaptation_reasoning(
                context_analysis, query_analysis, prompt_strategy
            )
        }

def _analyze_context_quality(self, context: str, query: str) -> Dict[str, Any]:
    """Analyze the quality and characteristics of retrieved context."""

    # Basic metrics
    word_count = len(context.split())
    source_count = context.count('[Source:')

    # Quality assessment using LLM
    quality_prompt = f"""
    Analyze the quality of this context for answering the given query:

    Query: {query}
    Context: {context[:1000]}...

    Assess:
    1. Relevance (0-10): How well does context address the query?
    2. Completeness (0-10): How comprehensive is the information?
    3. Clarity (0-10): How clear and well-organized is the information?
    4. Reliability (0-10): How credible and authoritative are the sources?

    Provide scores and brief explanations:
    """

def _analyze_query_characteristics(self, query: str) -> Dict[str, Any]:
    """Analyze query characteristics for prompt adaptation."""

    analysis_prompt = f"""
    Analyze this query's characteristics:

    Query: {query}

    Classify:
    1. Complexity (simple/moderate/complex): Based on number of concepts and relationships
    2. Type (factual/analytical/procedural/creative): Primary question type
    3. Scope (narrow/broad): How focused or wide-ranging is the question
    4. Reasoning Required (low/medium/high): Level of synthesis needed

    Return structured analysis:
    """

    try:
        response = self.llm_model.predict(analysis_prompt)
        return self._parse_query_analysis(response)
    except:
        return {
            'complexity': 'moderate',
            'type': 'factual',
            'scope': 'narrow',
            'reasoning_required': 'medium'
        }

def _select_prompt_strategy(self, context_analysis: Dict,
                           query_analysis: Dict,
                           retrieval_metadata: Dict) -> Dict[str, Any]:
    """Select optimal prompt strategy based on analysis."""

    strategy = {
        'base_template': 'factual_qa',
        'optimizations': [],
        'reasoning_depth': 'basic'
    }

    # Adapt based on query complexity
    if query_analysis.get('complexity') == 'complex':
        strategy['optimizations'].append('multi_step_reasoning')
        strategy['reasoning_depth'] = 'advanced'

    # Adapt based on context quality
    context_quality = context_analysis.get('overall_quality', 0.5)
    if context_quality < 0.6:
        strategy['optimizations'].extend(['context_analysis', 'uncertainty_handling'])

    # Always add confidence calibration for RAG
    if 'confidence_calibration' not in strategy['optimizations']:
        strategy['optimizations'].append('confidence_calibration')

    # Adapt template based on query type
    query_type = query_analysis.get('type', 'factual')
    if query_type in ['analytical', 'procedural', 'technical']:
        strategy['base_template'] = query_type

    return strategy

# Test the advanced prompt engineering system
engineer = RAGPromptEngineer(llm_model)
adapter = DynamicPromptAdapter(llm_model)

# Test with complex analytical query
complex_query = "Compare the advantages and disadvantages of microservices vs monolithic architecture for enterprise applications"
complex_context = "... comprehensive context about both architectures ..."

# Generate adaptive prompt
adaptive_result = adapter.adapt_prompt_dynamically(
    complex_query, complex_context, {'source_diversity': 0.8}
)

# Test with factual query and limited context
factual_query = "What is the capital of France?"
limited_context = "France is a country in Europe..."

factual_result = engineer.engineer_rag_prompt(
    factual_query, limited_context,
    query_type='factual_qa',
    optimizations=['confidence_calibration', 'context_analysis']
)

print("Advanced Prompting Results:")
print(f"Adaptive strategy: {adaptive_result['strategy']}")
print(f"Prompt complexity: {factual_result['complexity_score']}")
print(f"Optimizations applied: {factual_result['optimizations_applied']}")

class ProductionRAGSystem:
    """Production RAG system with advanced prompt engineering."""

    def __init__(self, llm_model, vector_store, tokenizer):
        self.hyde_enhancer = HyDEQueryEnhancer(llm_model, embedding_model)
        self.context_optimizer = ContextWindowOptimizer(tokenizer)
        self.prompt_adapter = DynamicPromptAdapter(llm_model)
        self.vector_store = vector_store
        self.llm_model = llm_model

    def query(self, user_query: str) -> Dict[str, Any]:
        """Complete RAG pipeline with advanced prompt engineering."""

        # Enhanced query processing
        hyde_result = self.hyde_enhancer.enhance_query_with_hyde(user_query)

        # Retrieval with enhanced query
        retrieved_chunks = self.vector_store.similarity_search_by_vector(
            hyde_result['enhanced_embedding'], k=20
        )

        # Context optimization
        optimized_context = self.context_optimizer.optimize_context_window(
            user_query, retrieved_chunks
        )

        # Adaptive prompt engineering
        prompt_result = self.prompt_adapter.adapt_prompt_dynamically(
            user_query, optimized_context['optimized_context'],
            {'retrieval_quality': len(retrieved_chunks)}
        )

        # Generate response
        response = self.llm_model.predict(prompt_result['adapted_prompt'])

        return {
            'response': response,
            'confidence_indicators': self._extract_confidence_indicators(response),
            'context_quality': optimized_context['efficiency_score'],
            'prompt_strategy': prompt_result['strategy'],
            'enhancement_metadata': hyde_result
        }