### Setup Weaviate and Reranker Clients using Python

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Demonstrates how to initialize synchronous and asynchronous Weaviate clients, as well as synchronous and asynchronous reranker clients (e.g., Cohere). This setup is crucial for interacting with Weaviate and external reranking services.

```python
from retrieve_dspy import clients

# Get synchronous Weaviate client
weaviate_client = clients.get_weaviate_client()

# Get asynchronous Weaviate client
async def setup_async():
    weaviate_async_client = await clients.get_and_connect_weaviate_async_client()
    # Use client...
    await weaviate_async_client.close()

# Get Cohere reranker clients
cohere_client = clients.get_cohere_client()
cohere_async_client = clients.get_cohere_async_client()

print(f"Cohere client: {cohere_client.name}")  # "cohere"
print(f"Client type: {type(cohere_client.client)}")
```

--------------------------------

### Asynchronous Weaviate Hybrid Search

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Provides an asynchronous implementation of Weaviate hybrid search using `async_weaviate_search_tool`. This example demonstrates connecting to Weaviate asynchronously, executing a search, and closing the connection.

```python
import os
import weaviate
from retrieve_dspy.database.weaviate_database import async_weaviate_search_tool
import asyncio

async def search_async():
    async_client = weaviate.use_async_with_weaviate_cloud(
        cluster_url=os.getenv("WEAVIATE_URL"),
        auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
    )
    await async_client.connect()

    results = await async_weaviate_search_tool(
        weaviate_async_client=async_client,
        query="How do I use Weaviate with Langchain?",
        collection_name="FreshstackLangchain",
        target_property_name="docs_text",
        retrieved_k=10,
        return_vector=False
    )

    await async_client.close()
    return results

results = asyncio.run(search_async())
```

--------------------------------

### Initialize Voyage AI Reranker Client

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Demonstrates how to obtain synchronous and asynchronous clients for Voyage AI's reranking service, which can then be used with DSPy's reranker implementations.

```python
voyage_client = clients.get_voyage_client()
voyage_async_client = clients.get_voyage_async_client()
```

--------------------------------

### Configure CrossEncoderReranker with Voyage AI

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Shows how to initialize a CrossEncoderReranker from the retrieve-dspy library, specifying collection details, the target property, the Weaviate client, and a list of reranker clients including Voyage AI.

```python
from retrieve_dspy import CrossEncoderReranker

reranker = CrossEncoderReranker(
    collection_name="MyCollection",
    target_property_name="content",
    weaviate_client=weaviate_client,
    reranker_clients=[cohere_client, voyage_client]
)
```

--------------------------------

### Populate Weaviate Database with Datasets using Python

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Provides utilities to load different benchmark datasets, such as Enron emails, BEIR, and LOTTE, into Weaviate collections. It requires a Weaviate client and a list of objects formatted according to the dataset structure.

```python
import os
import weaviate
from retrieve_dspy.datasets.populate_db import database_loader

# Initialize Weaviate client
weaviate_client = weaviate.connect_to_weaviate_cloud(
    cluster_url=os.getenv("WEAVIATE_URL"),
    auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
)

# Load Enron emails dataset
enron_objects = [
    {
        "email_body": "Email content here...",
        "dataset_id": "enron_001"
    },
    # ... more emails
]

database_loader(
    weaviate_client=weaviate_client,
    dataset_name="enron",
    objects=enron_objects
)

# Load BEIR dataset
beir_objects = [
    {
        "title": "Document title",
        "content": "Document content...",
        "doc_id": "doc_001"
    },
    # ... more documents
]

database_loader(
    weaviate_client=weaviate_client,
    dataset_name="beir/scifact",
    objects=beir_objects
)

# Load LOTTE dataset
lotte_objects = [
    {
        "text": "Document text...",
        "doc_id": "lotte_001"
    },
    # ... more documents
]

database_loader(
    weaviate_client=weaviate_client,
    dataset_name="lotte/lifestyle",
    objects=lotte_objects
)

print("Database populated successfully")
weaviate_client.close()
```

--------------------------------

### Multi-Query Retrieval with DSPy (Python)

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Generates multiple search queries from a single input question using an LLM, then retrieves documents for each query to enhance recall. Supports asynchronous execution and returns generated queries along with retrieved sources and usage statistics.

```python
import os
import dspy
from retrieve_dspy import MultiQueryWriter

# Configure DSPy with OpenAI
lm = dspy.LM("openai/gpt-4-mini", api_key=os.getenv("OPENAI_API_KEY"))
dspy.configure(lm=lm, track_usage=True)

# Create multi-query writer
retriever = MultiQueryWriter(
    collection_name="FreshstackLangchain",
    target_property_name="docs_text",
    retrieved_k=10,
    search_with_queries_concatenated=False,  # Search each query separately
    verbose=True
)

# Execute search - generates multiple queries automatically
question = "How do I integrate Weaviate and Langchain?"
response = retriever.forward(question=question)

print(f"Generated {len(response.searches)} search queries:")
for query in response.searches:
    print(f"  - {query}")

print(f"\nRetrieved {len(response.sources)} total documents")
print(f"Token usage: {response.usage}")

# Async execution with parallel searches
async def multi_query_async():
    response = await retriever.aforward(question=question)
    return response

result = asyncio.run(multi_query_async())
```

--------------------------------

### QUIPLER - Query Expansion with Parallel Reranking and RRF

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Implements query expansion by generating multiple query variations, performing parallel cross-encoder reranking on these variations, and then fusing the results using RRF. This approach aims for high retrieval quality by leveraging both query diversity and efficient reranking. It supports both synchronous and asynchronous execution.

```python
import os
import cohere
import weaviate
from retrieve_dspy import QUIPLER
from retrieve_dspy.models import RerankerClient
import asyncio

# Initialize clients
weaviate_client = weaviate.connect_to_weaviate_cloud(
    cluster_url=os.getenv("WEAVIATE_URL"),
    auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
)
cohere_client = cohere.ClientV2(api_key=os.getenv("COHERE_API_KEY"))

# Create QUIPLER retriever
quipler = QUIPLER(
    collection_name="EnronEmails",
    target_property_name="email_body",
    weaviate_client=weaviate_client,
    retrieved_k=50,
    reranked_k=20,
    rrf_k=60,
    verbose=True
)

# Execute with parallel search and reranking
question = "What are the implications of SBX12?"
response = quipler.forward(
    question=question,
    weaviate_client=weaviate_client,
    reranker_clients=[RerankerClient(name="cohere", client=cohere_client)]
)

print(f"Generated queries: {response.searches}")
print(f"Final fused results: {len(response.sources)} documents")

# Async execution for true parallelism
async def quipler_async():
    weaviate_async_client = weaviate.use_async_with_weaviate_cloud(
        cluster_url=os.getenv("WEAVIATE_URL"),
        auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
    )
    await weaviate_async_client.connect()

    cohere_async_client = cohere.AsyncClientV2(api_key=os.getenv("COHERE_API_KEY"))

    response = await quipler.aforward(
        question=question,
        weaviate_async_client=weaviate_async_client,
        reranker_clients=[RerankerClient(name="cohere", client=cohere_async_client)]
    )

    await weaviate_async_client.close()
    return response

result = asyncio.run(quipler_async())

```

--------------------------------

### RAGFusion - Query Expansion with Reciprocal Rank Fusion

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Expands a single query into multiple variations, retrieves documents for each variation, and then fuses the results using Reciprocal Rank Fusion (RRF). This method improves retrieval by considering different facets of the original query. It requires a Weaviate client and outputs the fused search results.

```python
import os
import weaviate
from retrieve_dspy import RAGFusion

# Initialize Weaviate client
weaviate_client = weaviate.connect_to_weaviate_cloud(
    cluster_url=os.getenv("WEAVIATE_URL"),
    auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
)

# Create RAG Fusion retriever
rag_fusion = RAGFusion(
    weaviate_client=weaviate_client,
    collection_name="EnronEmails",
    target_property_name="email_body",
    retrieved_k=20,   # Docs per query
    reranked_k=20,    # Final fused results
    rrf_k=60,         # RRF constant
    verbose=True,
    verbose_signature=True
)

# Execute search
question = "What are the implications of SBX12?"
response = rag_fusion.forward(question=question, weaviate_client=weaviate_client)

print(f"Generated {len(response.searches)} search queries:")
for i, query in enumerate(response.searches, 1):
    print(f"  {i}. {query}")

print(f"\nFused {len(response.sources)} unique documents")
for i, source in enumerate(response.sources[:3], 1):
    print(f"{i}. Score: {source.relevance_score:.4f}")
    print(f"   Source query: {source.source_query}")
    print(f"   Content: {source.content[:100]}...")

weaviate_client.close()
```

--------------------------------

### Evaluate Retrieval Metrics using Python

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Calculates various retrieval evaluation metrics such as Recall@k, nDCG@k, and Success@k. It takes ground truth target IDs and a list of retrieved documents as input. This can also be used to create metric functions for DSPy's evaluation framework.

```python
from retrieve_dspy import metrics
from retrieve_dspy.models import ObjectFromDB
from dspy import Example

# Sample retrieved documents
retrieved_docs = [
    ObjectFromDB(object_id="doc1", content="...", relevance_score=0.95),
    ObjectFromDB(object_id="doc2", content="...", relevance_score=0.87),
    ObjectFromDB(object_id="doc3", content="...", relevance_score=0.82)
]

target_ids = ["doc1", "doc5", "doc10"]

# Calculate Recall@k
recall = metrics.calculate_recall_at_k(
    target_ids=target_ids,
    retrieved_objects=retrieved_docs,
    k=5,
    verbose=True
)
print(f"Recall@5: {recall:.3f}")

# Calculate nDCG@k (normalized Discounted Cumulative Gain)
ndcg = metrics.calculate_nDCG_at_k(
    target_ids=target_ids,
    retrieved_objects=retrieved_docs,
    k=5,
    verbose=True
)
print(f"nDCG@5: {ndcg:.3f}")

# Calculate Success@k (Hit Rate)
success = metrics.calculate_success_at_k(
    target_ids=target_ids,
    retrieved_objects=retrieved_docs,
    k=5,
    verbose=True
)
print(f"Success@5: {success}")

# Create metric functions for DSPy evaluation
recall_metric = metrics.create_metric(metric_type="recall", k=10, verbose=False)
ndcg_metric = metrics.create_metric(metric_type="nDCG", k=10, verbose=False)

# Use with DSPy Evaluate
example = Example(dataset_ids=target_ids)
prediction = type('obj', (object,), {'sources': retrieved_docs})()

score = recall_metric(example, prediction)
print(f"Metric score: {score}")
```

--------------------------------

### CrossEncoderReranker - Neural Reranking with Cohere/Voyage AI

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Retrieves documents using an initial search and then reranks them with cross-encoder models from Cohere or Voyage AI. It requires Weaviate, Cohere, and optionally Voyage AI clients. The input is a question, and the output is a list of reranked documents with relevance scores.

```python
import os
import cohere
import weaviate
from retrieve_dspy import CrossEncoderReranker
from retrieve_dspy.models import RerankerClient

# Initialize clients
weaviate_client = weaviate.connect_to_weaviate_cloud(
    cluster_url=os.getenv("WEAVIATE_URL"),
    auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
)
cohere_client = cohere.ClientV2(api_key=os.getenv("COHERE_API_KEY"))

# Create reranker
reranker = CrossEncoderReranker(
    collection_name="EnronEmails",
    target_property_name="email_body",
    weaviate_client=weaviate_client,
    retrieved_k=50,  # Initial retrieval
    reranked_k=20,   # Final reranked results
    verbose=True,
    model_name_overrides={"cohere": "rerank-v3.5"}
)

# Execute with reranking
question = "What are the implications of SBX12?"
response = reranker.forward(
    question=question,
    weaviate_client=weaviate_client,
    reranker_clients=[RerankerClient(name="cohere", client=cohere_client)]
)

print(f"Reranked {len(response.sources)} documents")
for i, source in enumerate(response.sources[:5], 1):
    print(f"{i}. Score: {source.relevance_score:.4f} | ID: {source.object_id}")

# Mixture of Cross Encoders with RRF
import voyageai

voyage_client = voyageai.Client(api_key=os.getenv("VOYAGE_API_KEY"))

response_multi = reranker.forward(
    question=question,
    weaviate_client=weaviate_client,
    reranker_clients=[
        RerankerClient(name="cohere", client=cohere_client),
        RerankerClient(name="voyage", client=voyage_client)
    ]
)
```

--------------------------------

### Hybrid Search with Weaviate (Python)

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Performs hybrid search combining vector similarity and keyword matching using Weaviate's native capabilities. Requires Weaviate client initialization and returns structured response objects.

```python
import os
import weaviate
from retrieve_dspy import HybridSearch

# Initialize Weaviate client
weaviate_client = weaviate.connect_to_weaviate_cloud(
    cluster_url=os.getenv("WEAVIATE_URL"),
    auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
)

# Create hybrid search retriever
retriever = HybridSearch(
    collection_name="EnronEmails",
    target_property_name="email_body",
    weaviate_client=weaviate_client,
    retrieved_k=10,
    verbose=True
)

# Execute search
question = "What are the implications of SBX12?"
response = retriever.forward(question=question, weaviate_client=weaviate_client)

# Access results
for source in response.sources[:5]:
    print(f"ID: {source.object_id}")
    print(f"Score: {source.relevance_score}")
    print(f"Content: {source.content[:200]}...")
    print("---")

# Async execution
import asyncio

async def search_async():
    weaviate_async_client = weaviate.use_async_with_weaviate_cloud(
        cluster_url=os.getenv("WEAVIATE_URL"),
        auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
    )
    await weaviate_async_client.connect()

    response = await retriever.aforward(question=question, weaviate_async_client=weaviate_async_client)
    print(f"Found {len(response.sources)} documents")
    await weaviate_async_client.close()

asyncio.run(search_async())
```

--------------------------------

### Synchronous Weaviate Hybrid Search

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Implements a direct, low-level hybrid search in Weaviate using the `weaviate_search_tool`. This function connects to Weaviate, performs a search query, and returns results including object ID, relevance score, rank, content, and optionally the vector.

```python
import os
import weaviate
from retrieve_dspy.database.weaviate_database import weaviate_search_tool

weaviate_client = weaviate.connect_to_weaviate_cloud(
    cluster_url=os.getenv("WEAVIATE_URL"),
    auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
)

results = weaviate_search_tool(
    weaviate_client=weaviate_client,
    query="How do I use Weaviate with Langchain?",
    collection_name="FreshstackLangchain",
    target_property_name="docs_text",
    retrieved_k=10,
    return_vector=True
)

for result in results:
    print(f"ID: {result.object_id}")
    print(f"Score: {result.relevance_score}")
    print(f"Rank: {result.relevance_rank}")
    print(f"Content: {result.content[:100]}...")
    if result.vector:
        print(f"Vector dimensions: {len(result.vector)}")
    print("---")
```

--------------------------------

### HyDE Query Expansion with Weaviate (Python)

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Utilizes Hypothetical Document Embeddings (HyDE) by generating a hypothetical answer passage with an LLM, then performing a search using this passage. Requires Weaviate client and returns search passages and retrieved document details.

```python
import os
import weaviate
from retrieve_dspy import HyDE_QueryExpander

# Initialize clients
weaviate_client = weaviate.connect_to_weaviate_cloud(
    cluster_url=os.getenv("WEAVIATE_URL"),
    auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
)

# Create HyDE retriever
retriever = HyDE_QueryExpander(
    collection_name="BrightBiology",
    target_property_name="content",
    weaviate_client=weaviate_client,
    retrieved_k=20,
    verbose=True
)

# Execute search
question = "How many cells are in the human body?"
response = retriever.forward(question=question, weaviate_client=weaviate_client)

print(f"Hypothetical passage used for search: {response.searches[0]}")
print(f"Retrieved {len(response.sources)} documents")

for i, source in enumerate(response.sources[:3], 1):
    print(f"\n{i}. Score: {source.relevance_score:.4f}")
    print(f"   Content: {source.content[:300]}...")
```

--------------------------------

### Multi-Hop Retrieval with Cross-Encoder using Python

Source: https://context7.com/weaviate/retrieve-dspy/llms.txt

Performs iterative multi-hop retrieval by generating follow-up queries based on initial results. It then applies a cross-encoder for final reranking of documents. This function requires Weaviate and Voyage AI clients and returns ranked document sources.

```python
import os
import voyageai
import weaviate
from retrieve_dspy import SimplifiedBaleenWithCrossEncoder
from retrieve_dspy.models import RerankerClient

# Initialize clients
weaviate_client = weaviate.connect_to_weaviate_cloud(
    cluster_url=os.getenv("WEAVIATE_URL"),
    auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY"))
)
voyage_client = voyageai.Client(api_key=os.getenv("VOYAGE_API_KEY"))

# Create multi-hop retriever
retriever = SimplifiedBaleenWithCrossEncoder(
    weaviate_client=weaviate_client,
    collection_name="EnronEmails",
    target_property_name="email_body",
    reranker_clients=[RerankerClient(name="voyage", client=voyage_client)],
    retrieved_k=5,
    max_hops=2,
    reranked_N=20,
    verbose=True,
    verbose_signature=True
)

# Execute multi-hop search
question = "What are the implications of SBX12?"
response = retriever.forward(question=question)

print(f"Performed {2} hops of retrieval")
print(f"Final results: {len(response.sources)} documents after reranking")

for i, source in enumerate(response.sources[:5], 1):
    print(f"{i}. {source.content[:150]}...")
```

=== COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.