import os
import time
from typing import List, Optional
from dotenv import load_dotenv
from langchain_community.document_loaders import DirectoryLoader, ConfluenceLoader
from langchain_text_splitters import MarkdownTextSplitter, RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings
from langchain_pinecone import PineconeVectorStore
from pinecone import Pinecone, ServerlessSpec, PineconeException
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type

# Load environment variables from .env file
load_dotenv()

# Configuration constants
PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
INDEX_NAME = 'internal-wiki-rag-2-0'
CHUNK_SIZE = 1024
CHUNK_OVERLAP = 256
EMBEDDING_MODEL = 'text-embedding-3-small'
CONFLUENCE_URL = os.getenv('CONFLUENCE_URL')
CONFLUENCE_USERNAME = os.getenv('CONFLUENCE_USERNAME')
CONFLUENCE_API_TOKEN = os.getenv('CONFLUENCE_API_TOKEN')
WIKI_DIR = './internal-wiki-markdown'  # Local markdown export fallback

@retry(
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=4, max=60),
    retry=retry_if_exception_type((PineconeException, ConnectionError))
)
def initialize_pinecone_index(pc: Pinecone) -> None:
    \"\"\"Create Pinecone index if it does not exist, with hybrid sparse-dense support.\"\"\"
    try:
        if INDEX_NAME not in pc.list_indexes().names():
            print(f'Creating new Pinecone index: {INDEX_NAME}')
            pc.create_index(
                name=INDEX_NAME,
                dimension=1536,  # text-embedding-3-small dimension
                metric='dotproduct',  # Optimal for OpenAI embeddings
                spec=ServerlessSpec(
                    cloud='aws',
                    region='us-east-1'
                ),
                # Enable hybrid sparse-dense indexing (Pinecone 1.10 feature)
                index_type='hybrid'
            )
            # Wait for index to be ready
            while not pc.describe_index(INDEX_NAME).status['ready']:
                print('Waiting for index to initialize...')
                time.sleep(5)
            print(f'Index {INDEX_NAME} initialized successfully')
        else:
            print(f'Using existing Pinecone index: {INDEX_NAME}')
    except PineconeException as e:
        print(f'Pinecone index initialization failed: {e}')
        raise

def load_wiki_documents() -> List:
    \"\"\"Load documents from Confluence or local markdown fallback.\"\"\"
    documents = []
    # Try Confluence first if credentials are present
    if all([CONFLUENCE_URL, CONFLUENCE_USERNAME, CONFLUENCE_API_TOKEN]):
        print('Loading documents from Confluence...')
        try:
            loader = ConfluenceLoader(
                url=CONFLUENCE_URL,
                username=CONFLUENCE_USERNAME,
                api_token=CONFLUENCE_API_TOKEN,
                space_key='ENG',  # Engineering space key
                limit=10000  # Max documents to load
            )
            documents = loader.load()
            print(f'Loaded {len(documents)} documents from Confluence')
        except Exception as e:
            print(f'Confluence load failed: {e}, falling back to local markdown')

    # Fallback to local markdown directory
    if not documents:
        print(f'Loading documents from local directory: {WIKI_DIR}')
        loader = DirectoryLoader(
            WIKI_DIR,
            glob='**/*.md',
            use_multithreading=True,
            show_progress=True
        )
        documents = loader.load()
        print(f'Loaded {len(documents)} documents from local markdown')

    return documents

def main():
    # Validate required environment variables
    missing_vars = []
    if not PINECONE_API_KEY:
        missing_vars.append('PINECONE_API_KEY')
    if not OPENAI_API_KEY:
        missing_vars.append('OPENAI_API_KEY')
    if missing_vars:
        raise ValueError(f'Missing required environment variables: {', '.join(missing_vars)}')

    # Initialize Pinecone client (1.10 SDK)
    try:
        pc = Pinecone(api_key=PINECONE_API_KEY)
    except PineconeException as e:
        raise RuntimeError(f'Failed to initialize Pinecone client: {e}')

    # Initialize index
    initialize_pinecone_index(pc)

    # Load wiki documents
    documents = load_wiki_documents()
    if not documents:
        raise RuntimeError('No documents loaded from any source')

    # Split documents into chunks (wiki-optimized splitter)
    # Use MarkdownTextSplitter for markdown content to preserve headings/code blocks
    splitter = MarkdownTextSplitter(
        chunk_size=CHUNK_SIZE,
        chunk_overlap=CHUNK_OVERLAP
    )
    # Fallback to recursive splitter for non-markdown content
    if not any(doc.metadata.get('source', '').endswith('.md') for doc in documents):
        splitter = RecursiveCharacterTextSplitter(
            chunk_size=CHUNK_SIZE,
            chunk_overlap=CHUNK_OVERLAP,
            separators=['\\n\\n', '\\n', ' ', '']
        )

    chunks = splitter.split_documents(documents)
    print(f'Split {len(documents)} documents into {len(chunks)} chunks')

    # Initialize embeddings model
    embeddings = OpenAIEmbeddings(
        model=EMBEDDING_MODEL,
        openai_api_key=OPENAI_API_KEY
    )

    # Upsert chunks to Pinecone
    print(f'Upserting {len(chunks)} chunks to Pinecone index {INDEX_NAME}...')
    vector_store = PineconeVectorStore.from_documents(
        documents=chunks,
        embedding=embeddings,
        index_name=INDEX_NAME,
        # Pinecone 1.10 hybrid indexing requires sparse vector generation
        sparse_vector_generator='bm25'  # Built-in BM25 sparse vector support
    )
    print(f'Successfully upserted {len(chunks)} chunks to Pinecone')

if __name__ == '__main__':
    start_time = time.time()
    try:
        main()
    except Exception as e:
        print(f'Ingestion pipeline failed: {e}')
        raise
    finally:
        print(f'Total ingestion time: {time.time() - start_time:.2f} seconds')

import os
import time
from typing import Dict, List, Optional
from dotenv import load_dotenv
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_pinecone import PineconeVectorStore
from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import FlashrankRerank
from langchain.chains import RetrievalChain
from langchain.prompts import ChatPromptTemplate
from pinecone import Pinecone, PineconeException
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type

# Load environment variables
load_dotenv()

# Configuration
PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
INDEX_NAME = 'internal-wiki-rag-2-0'
LLM_MODEL = 'gpt-4o-2024-08-06'
EMBEDDING_MODEL = 'text-embedding-3-small'
TOP_K_RETRIEVAL = 20  # Retrieve 20 initial results
TOP_K_RERANK = 5  # Return top 5 reranked results
CONFIDENCE_THRESHOLD = 0.7  # Minimum rerank score to return answer

# RAG 2.0 system prompt optimized for internal wiki queries
SYSTEM_PROMPT = '''You are an internal engineering wiki assistant. Use the provided context to answer the user's question.
If the context does not contain the answer, say \"I don't have enough information in the internal wiki to answer this question.\"
Do not make up information. Cite the source URL or page title for every claim you make.
Use technical terminology consistent with our internal docs. For code questions, provide runnable examples if present in the context.'''

USER_PROMPT = '''Context:
{context}

Question: {question}

Answer:'''

@retry(
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=2, max=30),
    retry=retry_if_exception_type((PineconeException, ConnectionError))
)
def initialize_vector_store() -> PineconeVectorStore:
    \"\"\"Initialize Pinecone vector store with hybrid retrieval enabled.\"\"\"
    try:
        pc = Pinecone(api_key=PINECONE_API_KEY)
        # Verify index exists and is hybrid
        index = pc.describe_index(INDEX_NAME)
        if index.index_type != 'hybrid':
            raise ValueError(f'Index {INDEX_NAME} is not a hybrid index. Please recreate with index_type=\"hybrid\"')

        embeddings = OpenAIEmbeddings(
            model=EMBEDDING_MODEL,
            openai_api_key=OPENAI_API_KEY
        )

        vector_store = PineconeVectorStore(
            index_name=INDEX_NAME,
            embedding=embeddings,
            # Enable hybrid sparse-dense retrieval (Pinecone 1.10)
            sparse_vector_generator='bm25'
        )
        print(f'Initialized hybrid vector store for index {INDEX_NAME}')
        return vector_store
    except PineconeException as e:
        print(f'Vector store initialization failed: {e}')
        raise

def build_rag_chain(vector_store: PineconeVectorStore) -> RetrievalChain:
    \"\"\"Build RAG 2.0 chain with hybrid retrieval and reranking.\"\"\"
    # Base retriever: hybrid sparse + dense, get top 20 results
    base_retriever = vector_store.as_retriever(
        search_type='hybrid',  # Pinecone 1.10 hybrid search
        search_kwargs={
            'k': TOP_K_RETRIEVAL,
            'alpha': 0.5  # Balance between dense (1.0) and sparse (0.0) search
        }
    )

    # Reranker: FlashRank (lightweight, 12ms latency per query)
    reranker = FlashrankRerank(
        model_name='ms-marco-MiniLM-L-12-v2',
        top_n=TOP_K_RERANK
    )

    # Compressed retriever with reranking
    compression_retriever = ContextualCompressionRetriever(
        base_retriever=base_retriever,
        base_compressor=reranker
    )

    # LLM for answer generation
    llm = ChatOpenAI(
        model=LLM_MODEL,
        openai_api_key=OPENAI_API_KEY,
        temperature=0.0  # Deterministic answers for wiki queries
    )

    # Prompt template
    prompt = ChatPromptTemplate.from_messages([
        ('system', SYSTEM_PROMPT),
        ('human', USER_PROMPT)
    ])

    # Build retrieval chain (LangChain 0.3 RetrievalChain)
    chain = RetrievalChain(
        retriever=compression_retriever,
        combine_docs_chain=prompt | llm
    )

    return chain

def query_rag_pipeline(chain: RetrievalChain, question: str) -> Dict:
    \"\"\"Query the RAG 2.0 pipeline and return answer with metadata.\"\"\"
    start_time = time.time()
    try:
        # Invoke chain with question
        result = chain.invoke({'query': question})

        # Extract rerank scores from retriever metadata
        retrieved_docs = chain.retriever.get_relevant_documents(question)
        rerank_scores = [doc.metadata.get('rerank_score', 0.0) for doc in retrieved_docs]
        avg_rerank_score = sum(rerank_scores) / len(rerank_scores) if rerank_scores else 0.0

        # Check confidence threshold
        if avg_rerank_score < CONFIDENCE_THRESHOLD:
            result['answer'] = f'Low confidence ({avg_rerank_score:.2f}) in retrieved context. {result['answer']}'

        # Add latency and metadata
        result['latency_ms'] = (time.time() - start_time) * 1000
        result['retrieved_docs_count'] = len(retrieved_docs)
        result['avg_rerank_score'] = avg_rerank_score
        result['sources'] = [doc.metadata.get('source', 'unknown') for doc in retrieved_docs]

        return result
    except Exception as e:
        print(f'Query failed: {e}')
        return {
            'answer': f'Pipeline error: {str(e)}',
            'latency_ms': (time.time() - start_time) * 1000,
            'retrieved_docs_count': 0,
            'avg_rerank_score': 0.0,
            'sources': []
        }

def main():
    # Validate env vars
    if not PINECONE_API_KEY or not OPENAI_API_KEY:
        raise ValueError('Missing PINECONE_API_KEY or OPENAI_API_KEY')

    # Initialize vector store and chain
    vector_store = initialize_vector_store()
    rag_chain = build_rag_chain(vector_store)

    # Example query
    test_question = 'How do I configure the internal gRPC timeout for the payment service?'
    print(f'Querying: {test_question}')

    result = query_rag_pipeline(rag_chain, test_question)

    print(f'\nAnswer: {result['answer']}')
    print(f'Latency: {result['latency_ms']:.2f}ms')
    print(f'Retrieved {result['retrieved_docs_count']} docs, avg rerank score: {result['avg_rerank_score']:.2f}')
    print(f'Sources: {result['sources']}')

if __name__ == '__main__':
    start_time = time.time()
    try:
        main()
    except Exception as e:
        print(f'Retrieval pipeline failed: {e}')
        raise
    finally:
        print(f'Total runtime: {time.time() - start_time:.2f} seconds')

import os
import json
import time
from typing import List, Dict
from dotenv import load_dotenv
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_pinecone import PineconeVectorStore
from langchain.evaluation import load_evaluator, EvaluatorType
from ragas import evaluate
from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    context_precision,
    context_recall
)
from datasets import Dataset
from pinecone import Pinecone

# Load environment variables
load_dotenv()

# Configuration
PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')
INDEX_NAME = 'internal-wiki-rag-2-0'
LLM_MODEL = 'gpt-4o-2024-08-06'
EMBEDDING_MODEL = 'text-embedding-3-small'
EVAL_DATASET_PATH = './eval-dataset.json'  # Ground truth Q&A pairs
RESULTS_PATH = './rag-eval-results.json'

# Evaluation dataset schema: list of {question, ground_truth_answer, ground_truth_context}
SAMPLE_EVAL_DATA = [
    {
        \"question\": \"What is the internal deployment process for backend services?\",
        \"ground_truth_answer\": \"Backend services are deployed via ArgoCD with canary rollouts: 10% traffic for 1 hour, 50% for 2 hours, 100% after approval.\",
        \"ground_truth_context\": [\"deployments/backend.md\", \"argocd-config.md\"]
    },
    {
        \"question\": \"How do I get access to the production database?\",
        \"ground_truth_answer\": \"Submit a Jira ticket to the DB team with manager approval, enable MFA, use the read-only credential vault.\",
        \"ground_truth_context\": [\"security/database-access.md\"]
    },
    {
        \"question\": \"What is the timeout for the internal payment API?\",
        \"ground_truth_answer\": \"The internal payment API has a 5s timeout for GET requests, 10s for POST requests, configurable via the PAYMENT_TIMEOUT env var.\",
        \"ground_truth_context\": [\"services/payment.md\"]
    }
]

def load_eval_dataset() -> List[Dict]:
    \"\"\"Load or create evaluation dataset.\"\"\"
    if os.path.exists(EVAL_DATASET_PATH):
        with open(EVAL_DATASET_PATH, 'r') as f:
            return json.load(f)
    else:
        print(f'Eval dataset not found, creating sample at {EVAL_DATASET_PATH}')
        with open(EVAL_DATASET_PATH, 'w') as f:
            json.dump(SAMPLE_EVAL_DATA, f, indent=2)
        return SAMPLE_EVAL_DATA

def initialize_rag_pipeline() -> PineconeVectorStore:
    \"\"\"Initialize RAG pipeline components for evaluation.\"\"\"
    pc = Pinecone(api_key=PINECONE_API_KEY)
    embeddings = OpenAIEmbeddings(
        model=EMBEDDING_MODEL,
        openai_api_key=OPENAI_API_KEY
    )
    vector_store = PineconeVectorStore(
        index_name=INDEX_NAME,
        embedding=embeddings,
        sparse_vector_generator='bm25'
    )
    return vector_store

def run_rag_inference(vector_store: PineconeVectorStore, question: str) -> Dict:
    \"\"\"Run RAG inference for a single question, return answer and context.\"\"\"
    # Retrieve context
    retriever = vector_store.as_retriever(
        search_type='hybrid',
        search_kwargs={'k': 5, 'alpha': 0.5}
    )
    context_docs = retriever.get_relevant_documents(question)
    context = [doc.page_content for doc in context_docs]
    sources = [doc.metadata.get('source', 'unknown') for doc in context_docs]

    # Generate answer
    llm = ChatOpenAI(
        model=LLM_MODEL,
        openai_api_key=OPENAI_API_KEY,
        temperature=0.0
    )
    prompt = f'''Use the following context to answer the question. If context is insufficient, say so.
Context: {chr(10).join(context)}
Question: {question}
Answer:'''
    answer = llm.invoke(prompt).content

    return {
        'question': question,
        'answer': answer,
        'context': context,
        'sources': sources
    }

def evaluate_with_langchain(inference_results: List[Dict]) -> Dict:
    \"\"\"Evaluate RAG results using LangChain built-in evaluators.\"\"\"
    qa_evaluator = load_evaluator(EvaluatorType.QA, llm=ChatOpenAI(model=LLM_MODEL, openai_api_key=OPENAI_API_KEY))
    faithfulness_evaluator = load_evaluator(EvaluatorType.FAITHFULNESS, llm=ChatOpenAI(model=LLM_MODEL, openai_api_key=OPENAI_API_KEY))

    qa_scores = []
    faithfulness_scores = []

    for result in inference_results:
        # QA correctness (compared to ground truth)
        qa_score = qa_evaluator.evaluate(
            prediction=result['answer'],
            reference=result['ground_truth_answer'],
            input=result['question']
        )['score']
        qa_scores.append(qa_score)

        # Faithfulness (answer grounded in context)
        faith_score = faithfulness_evaluator.evaluate(
            prediction=result['answer'],
            context=result['context']
        )['score']
        faithfulness_scores.append(faith_score)

    return {
        'langchain_qa_correctness': sum(qa_scores) / len(qa_scores),
        'langchain_faithfulness': sum(faithfulness_scores) / len(faithfulness_scores)
    }

def evaluate_with_ragas(inference_results: List[Dict]) -> Dict:
    \"\"\"Evaluate RAG results using Ragas metrics.\"\"\"
    # Prepare dataset for Ragas
    ragas_data = {
        'question': [r['question'] for r in inference_results],
        'answer': [r['answer'] for r in inference_results],
        'context': [r['context'] for r in inference_results],
        'ground_truth': [r['ground_truth_answer'] for r in inference_results]
    }
    dataset = Dataset.from_dict(ragas_data)

    # Run Ragas evaluation
    metrics = [faithfulness, answer_relevancy, context_precision, context_recall]
    results = evaluate(dataset, metrics=metrics)

    return {
        'ragas_faithfulness': results['faithfulness'],
        'ragas_answer_relevancy': results['answer_relevancy'],
        'ragas_context_precision': results['context_precision'],
        'ragas_context_recall': results['context_recall']
    }

def main():
    # Load eval dataset
    eval_data = load_eval_dataset()
    print(f'Loaded {len(eval_data)} evaluation samples')

    # Initialize RAG pipeline
    vector_store = initialize_rag_pipeline()

    # Run inference for all eval questions
    inference_results = []
    for sample in eval_data:
        print(f'Running inference for: {sample[\"question\"]}')
        result = run_rag_inference(vector_store, sample['question'])
        # Add ground truth from eval data
        result['ground_truth_answer'] = sample['ground_truth_answer']
        result['ground_truth_context'] = sample['ground_truth_context']
        inference_results.append(result)

    # Run evaluations
    print('Running LangChain evaluation...')
    langchain_results = evaluate_with_langchain(inference_results)

    print('Running Ragas evaluation...')
    ragas_results = evaluate_with_ragas(inference_results)

    # Combine results
    final_results = {**langchain_results, **ragas_results}
    final_results['total_samples'] = len(eval_data)
    final_results['timestamp'] = time.time()

    # Save results
    with open(RESULTS_PATH, 'w') as f:
        json.dump(final_results, f, indent=2)

    print(f'\nEvaluation Results:')
    for key, value in final_results.items():
        if isinstance(value, float):
            print(f'{key}: {value:.4f}')
        else:
            print(f'{key}: {value}')

if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        print(f'Evaluation pipeline failed: {e}')
        raise

def on_embedding_end(self, embeddings, **kwargs):
    # Track embedding token usage (OpenAI text-embedding-3-small: 1 token per character ~ roughly)
    self.total_embedding_tokens += sum(len(text) for text in kwargs.get('texts', []))

def on_llm_end(self, response, **kwargs):
    # Track LLM token usage
    self.total_llm_tokens += response.llm_output.get('token_usage', {}).get('total_tokens', 0)

def on_retriever_end(self, documents, **kwargs):
    # Track Pinecone read units (1 read per retrieved document)
    self.pinecone_reads += len(documents)

def get_cost(self):
    # Calculate cost: embeddings $0.02 per 1M tokens, LLM $0.15 per 1M input, $0.60 per 1M output, Pinecone $0.10 per 1M reads
    embedding_cost = (self.total_embedding_tokens / 1e6) * 0.02
    llm_cost = (self.total_llm_tokens / 1e6) * 0.15  # Simplified, assumes input only
    pinecone_cost = (self.pinecone_reads / 1e6) * 0.10
    return embedding_cost + llm_cost + pinecone_cost