Graph vs Vector Databases: When Relationships Meet Similarity¶

# Qdrant client setup
from qdrant_client import QdrantClient
from qdrant_client.http import models
import numpy as np

# Connect to Qdrant
client = QdrantClient("http://localhost:6333")

# Create collection
client.recreate_collection(
    collection_name="documents",
    vectors_config=models.VectorParams(
        size=384,  # Vector dimension
        distance=models.Distance.COSINE
    )
)

# Insert vectors
documents = [
    {"id": 1, "text": "Machine learning algorithms", "vector": [0.1] * 384},
    {"id": 2, "text": "Deep learning neural networks", "vector": [0.2] * 384},
    {"id": 3, "text": "Natural language processing", "vector": [0.3] * 384},
    {"id": 4, "text": "Computer vision applications", "vector": [0.4] * 384}
]

points = [
    models.PointStruct(
        id=doc["id"],
        vector=doc["vector"],
        payload={"text": doc["text"]}
    )
    for doc in documents
]

client.upsert(
    collection_name="documents",
    points=points
)

# Search for similar documents
search_result = client.search(
    collection_name="documents",
    query_vector=[0.15] * 384,  # Query vector
    limit=3,
    with_payload=True
)

for result in search_result:
    print(f"ID: {result.id}, Score: {result.score}, Text: {result.payload['text']}")

# Hybrid approach: Graph + Vector
# 1. Use vector DB for similarity search
similar_users = vector_db.search(
    query_vector=user_embedding,
    limit=100
)

# 2. Use graph DB for relationship context
for user_id in similar_users:
    # Check if users are connected in graph
    if graph_db.has_path(current_user, user_id, max_depth=2):
        # Add relationship context to recommendation
        recommendation = {
            "user_id": user_id,
            "similarity_score": user_score,
            "relationship_path": graph_db.get_path(current_user, user_id)
        }

# Complete Qdrant example
import numpy as np
from qdrant_client import QdrantClient
from qdrant_client.http import models

# Initialize client
client = QdrantClient("http://localhost:6333")

# Create collection
client.recreate_collection(
    collection_name="users",
    vectors_config=models.VectorParams(
        size=128,  # Embedding dimension
        distance=models.Distance.COSINE
    )
)

# Generate sample user embeddings
users = [
    {"id": 1, "name": "Alice", "interests": ["AI", "Data Science"]},
    {"id": 2, "name": "Bob", "interests": ["AI", "Machine Learning"]},
    {"id": 3, "name": "Charlie", "interests": ["Data Science", "Statistics"]},
    {"id": 4, "name": "Diana", "interests": ["Machine Learning", "Deep Learning"]}
]

# Create embeddings (simplified - in practice, use real embedding models)
embeddings = np.random.rand(4, 128).tolist()

# Insert users
points = [
    models.PointStruct(
        id=user["id"],
        vector=embeddings[i],
        payload={"name": user["name"], "interests": user["interests"]}
    )
    for i, user in enumerate(users)
]

client.upsert(
    collection_name="users",
    points=points
)

# Search for similar users
query_vector = np.random.rand(128).tolist()
search_result = client.search(
    collection_name="users",
    query_vector=query_vector,
    limit=3,
    with_payload=True
)

print("Similar users:")
for result in search_result:
    print(f"Name: {result.payload['name']}, Score: {result.score}")

# Hybrid recommendation system
class HybridRecommender:
    def __init__(self, graph_db, vector_db):
        self.graph_db = graph_db
        self.vector_db = vector_db

    def recommend_users(self, user_id, limit=10):
        # 1. Get user embedding from vector DB
        user_embedding = self.vector_db.get_user_embedding(user_id)

        # 2. Find similar users in vector space
        similar_users = self.vector_db.search(
            query_vector=user_embedding,
            limit=100
        )

        # 3. Filter by graph relationships
        recommendations = []
        for user in similar_users:
            # Check if users are connected in graph
            if self.graph_db.has_connection(user_id, user.id, max_depth=3):
                recommendations.append({
                    "user_id": user.id,
                    "similarity_score": user.score,
                    "connection_path": self.graph_db.get_connection_path(user_id, user.id)
                })

        return recommendations[:limit]

# ✅ DO: Normalize embeddings
def normalize_embedding(embedding):
    return embedding / np.linalg.norm(embedding)

# ❌ DON'T: Store raw embeddings without normalization
raw_embedding = [0.1, 0.2, 0.3, ...]  # Not normalized

# ✅ DO: Choose appropriate distance metrics
# Cosine: for normalized embeddings
# Euclidean: for raw embeddings
# Dot product: for similarity scores

# ❌ DON'T: Use wrong distance metrics
# Cosine distance on unnormalized vectors
# Euclidean distance on normalized vectors

# ✅ DO: Batch operations
def batch_insert_vectors(vectors, batch_size=1000):
    for i in range(0, len(vectors), batch_size):
        batch = vectors[i:i + batch_size]
        client.upsert(collection_name="vectors", points=batch)

# ❌ DON'T: Insert vectors one by one
for vector in vectors:
    client.upsert(collection_name="vectors", points=[vector])  # Inefficient

# ✅ DO: Use graph for explainability
def explain_recommendation(user_id, recommended_user_id):
    # Get similarity score from vector DB
    similarity = vector_db.get_similarity(user_id, recommended_user_id)

    # Get relationship path from graph DB
    path = graph_db.get_connection_path(user_id, recommended_user_id)

    return {
        "similarity_score": similarity,
        "connection_path": path,
        "explanation": f"Similar interests (score: {similarity}) and connected via {path}"
    }

# ❌ DON'T: Use graph for bulk similarity search
def bad_similarity_search():
    # This will be slow and inefficient
    all_users = graph_db.get_all_users()
    for user in all_users:
        similarity = calculate_similarity(current_user, user)
        if similarity > threshold:
            recommendations.append(user)

# ❌ WRONG: Storing embeddings in PostgreSQL arrays
# This will be slow and inefficient
CREATE TABLE users (
    id SERIAL PRIMARY KEY,
    name VARCHAR(100),
    embedding FLOAT[]  -- Don't do this!
);

# ✅ CORRECT: Use dedicated vector database
# Qdrant, Weaviate, Pinecone, etc.

# ❌ WRONG: Not normalizing embeddings
raw_embedding = [0.1, 0.2, 0.3, ...]  # Not normalized
client.upsert(collection_name="vectors", points=[raw_embedding])

# ✅ CORRECT: Normalize embeddings
normalized_embedding = normalize_embedding(raw_embedding)
client.upsert(collection_name="vectors", points=[normalized_embedding])

# ❌ WRONG: Using wrong distance metrics
# Cosine distance on unnormalized vectors
# Euclidean distance on normalized vectors

# ✅ CORRECT: Match distance metric to embedding type
# Cosine: for normalized embeddings
# Euclidean: for raw embeddings

# ❌ WRONG: Using graph for similarity search
def bad_similarity_search():
    # This will be extremely slow
    all_users = graph_db.get_all_users()
    similarities = []
    for user in all_users:
        similarity = calculate_similarity(current_user, user)
        similarities.append((user, similarity))
    return sorted(similarities, key=lambda x: x[1], reverse=True)

# ✅ CORRECT: Use vector DB for similarity search
def good_similarity_search():
    # Fast approximate search
    similar_users = vector_db.search(
        query_vector=current_user_embedding,
        limit=100
    )
    return similar_users

# ❌ WRONG: Using vector DB for relationship queries
def bad_relationship_query():
    # This won't work - vector DB doesn't understand relationships
    friends = vector_db.search(
        query_vector=user_embedding,
        filter={"relationship": "friend"}  # Not supported!
    )

# ✅ CORRECT: Use graph DB for relationship queries
def good_relationship_query():
    # Graph DB understands relationships
    friends = graph_db.get_friends(user_id)
    return friends

# ✅ DO: Choose appropriate index type
# HNSW: Fast approximate search
# IVF: Balanced performance
# PQ: Memory-efficient

# ✅ DO: Batch operations
def batch_upsert(vectors, batch_size=1000):
    for i in range(0, len(vectors), batch_size):
        batch = vectors[i:i + batch_size]
        client.upsert(collection_name="vectors", points=batch)

# ✅ DO: Use filters for pre-filtering
search_result = client.search(
    collection_name="vectors",
    query_vector=query_vector,
    query_filter=models.Filter(
        must=[
            models.FieldCondition(
                key="category",
                match=models.MatchValue(value="technology")
            )
        ]
    ),
    limit=10
)

# ✅ DO: Monitor index performance
def monitor_performance():
    collection_info = client.get_collection("vectors")
    print(f"Index status: {collection_info.status}")
    print(f"Vector count: {collection_info.vectors_count}")

# ✅ DO: Use vector DB for candidate generation
def hybrid_recommendation(user_id, limit=10):
    # Fast vector search for candidates
    candidates = vector_db.search(
        query_vector=user_embedding,
        limit=1000  # Get more candidates
    )

    # Graph filtering for context
    recommendations = []
    for candidate in candidates:
        if graph_db.has_connection(user_id, candidate.id, max_depth=2):
            recommendations.append(candidate)
            if len(recommendations) >= limit:
                break

    return recommendations

# ❌ DON'T: Use graph for similarity search
def bad_hybrid_recommendation(user_id, limit=10):
    # This will be extremely slow
    all_users = graph_db.get_all_users()
    similarities = []
    for user in all_users:
        similarity = calculate_similarity(user_id, user.id)
        similarities.append((user, similarity))
    return sorted(similarities, key=lambda x: x[1], reverse=True)[:limit]

# Monitor collection health
def monitor_collection_health():
    collection_info = client.get_collection("vectors")
    print(f"Collection status: {collection_info.status}")
    print(f"Vector count: {collection_info.vectors_count}")
    print(f"Indexed vector count: {collection_info.indexed_vectors_count}")

    # Check index performance
    if collection_info.indexed_vectors_count < collection_info.vectors_count:
        print("WARNING: Not all vectors are indexed!")

# Monitor search performance
import time

def benchmark_search(query_vector, iterations=100):
    start_time = time.time()

    for _ in range(iterations):
        client.search(
            collection_name="vectors",
            query_vector=query_vector,
            limit=10
        )

    end_time = time.time()
    avg_time = (end_time - start_time) / iterations
    print(f"Average search time: {avg_time:.3f} seconds")

# Monitor hybrid system performance
class HybridMonitor:
    def __init__(self, graph_db, vector_db):
        self.graph_db = graph_db
        self.vector_db = vector_db

    def monitor_system_health(self):
        # Graph DB health
        graph_health = self.graph_db.check_health()

        # Vector DB health
        vector_health = self.vector_db.check_health()

        # Hybrid performance
        hybrid_performance = self.benchmark_hybrid_queries()

        return {
            "graph_db": graph_health,
            "vector_db": vector_health,
            "hybrid_performance": hybrid_performance
        }

    def benchmark_hybrid_queries(self):
        # Test hybrid recommendation performance
        start_time = time.time()

        for user_id in self.get_test_users():
            self.hybrid_recommendation(user_id)

        end_time = time.time()
        return {
            "total_time": end_time - start_time,
            "queries_per_second": len(self.get_test_users()) / (end_time - start_time)
        }

# Qdrant security configuration
from qdrant_client.http import models

# Create collection with access control
client.create_collection(
    collection_name="private_documents",
    vectors_config=models.VectorParams(size=384, distance=models.Distance.COSINE),
    # Add access control
    hnsw_config=models.HnswConfigDiff(
        on_disk=True  # Store index on disk for security
    )
)

# Search with user context
def secure_search(user_id, query_vector, limit=10):
    # Get user permissions
    user_permissions = get_user_permissions(user_id)

    # Apply filters based on permissions
    search_filter = models.Filter(
        must=[
            models.FieldCondition(
                key="access_level",
                match=models.MatchValue(value=user_permissions["access_level"])
            )
        ]
    )

    return client.search(
        collection_name="private_documents",
        query_vector=query_vector,
        query_filter=search_filter,
        limit=limit
    )

# Hybrid security model
class HybridSecurity:
    def __init__(self, graph_db, vector_db):
        self.graph_db = graph_db
        self.vector_db = vector_db

    def secure_hybrid_query(self, user_id, query_vector):
        # Check graph permissions
        if not self.graph_db.has_permission(user_id, "read"):
            raise PermissionError("User lacks read permission")

        # Apply graph-based filtering
        allowed_entities = self.graph_db.get_allowed_entities(user_id)

        # Search vector DB with graph filters
        search_filter = models.Filter(
            must=[
                models.FieldCondition(
                    key="entity_id",
                    match=models.MatchAny(any=allowed_entities)
                )
            ]
        )

        return self.vector_db.search(
            query_vector=query_vector,
            query_filter=search_filter
        )

# Vector: Similarity search
similar_users = client.search(
    collection_name="users",
    query_vector=user_embedding,
    limit=10
)

# Vector: Batch insert
client.upsert(
    collection_name="users",
    points=user_points
)

# Hybrid: Graph + Vector
def hybrid_recommendation(user_id):
    # Vector similarity for candidates
    candidates = vector_db.search(user_embedding, limit=100)

    # Graph filtering for context
    recommendations = []
    for candidate in candidates:
        if graph_db.has_connection(user_id, candidate.id):
            recommendations.append(candidate)

    return recommendations