# python/01-connection-pooling.py"""Database connection pooling patterns and management"""fromtypingimportList,Dict,Any,Optional,Union,Tuplefromdataclassesimportdataclass,asdictfromenumimportEnumimportasyncioimporttimeimportthreadingfromcontextlibimportasynccontextmanager,contextmanagerfromdatetimeimportdatetime,timedeltaimportloggingfromqueueimportQueue,Emptyimportweakrefimportpsycopg2frompsycopg2importpoolfromsqlalchemyimportcreate_engine,textfromsqlalchemy.poolimportQueuePool,StaticPoolfromsqlalchemy.ormimportsessionmakerfromsqlalchemy.ext.asyncioimportcreate_async_engine,AsyncSessionimportasyncpgimportaioredisfromredisimportConnectionPoolasRedisConnectionPool# Configure logginglogging.basicConfig(level=logging.INFO)logger=logging.getLogger(__name__)classConnectionState(Enum):"""Connection state enumeration"""IDLE="idle"BUSY="busy"CLOSED="closed"ERROR="error"@dataclassclassConnectionMetrics:"""Connection metrics"""total_connections:int=0active_connections:int=0idle_connections:int=0failed_connections:int=0connection_requests:int=0connection_wait_time:float=0.0last_activity:datetime=NoneclassDatabaseConnectionPool:"""Advanced database connection pool"""def__init__(self,connection_string:str,min_connections:int=5,max_connections:int=20,connection_timeout:int=30):self.connection_string=connection_stringself.min_connections=min_connectionsself.max_connections=max_connectionsself.connection_timeout=connection_timeoutself.pool=Noneself.metrics=ConnectionMetrics()self.connection_lock=threading.Lock()self.connections={}self.initialize_pool()definitialize_pool(self)->None:"""Initialize connection pool"""try:self.pool=psycopg2.pool.ThreadedConnectionPool(minconn=self.min_connections,maxconn=self.max_connections,dsn=self.connection_string,connect_timeout=self.connection_timeout)logger.info(f"Connection pool initialized with {self.min_connections}-{self.max_connections} connections")exceptExceptionase:logger.error(f"Failed to initialize connection pool: {e}")raisedefget_connection(self)->psycopg2.extensions.connection:"""Get connection from pool"""start_time=time.time()try:connection=self.pool.getconn()ifconnection:self.metrics.active_connections+=1self.metrics.connection_requests+=1self.metrics.connection_wait_time+=time.time()-start_timeself.metrics.last_activity=datetime.utcnow()# Store connection referencewithself.connection_lock:self.connections[id(connection)]={'connection':connection,'state':ConnectionState.BUSY,'created_at':datetime.utcnow()}logger.debug(f"Connection acquired: {id(connection)}")returnconnectionelse:raiseException("Failed to get connection from pool")exceptExceptionase:self.metrics.failed_connections+=1logger.error(f"Failed to get connection: {e}")raisedefreturn_connection(self,connection:psycopg2.extensions.connection)->None:"""Return connection to pool"""try:connection_id=id(connection)withself.connection_lock:ifconnection_idinself.connections:self.connections[connection_id]['state']=ConnectionState.IDLEdelself.connections[connection_id]self.pool.putconn(connection)self.metrics.active_connections-=1self.metrics.idle_connections+=1logger.debug(f"Connection returned: {connection_id}")exceptExceptionase:logger.error(f"Failed to return connection: {e}")defclose_connection(self,connection:psycopg2.extensions.connection)->None:"""Close connection"""try:connection_id=id(connection)withself.connection_lock:ifconnection_idinself.connections:self.connections[connection_id]['state']=ConnectionState.CLOSEDdelself.connections[connection_id]connection.close()self.metrics.active_connections-=1logger.debug(f"Connection closed: {connection_id}")exceptExceptionase:logger.error(f"Failed to close connection: {e}")defget_pool_status(self)->Dict[str,Any]:"""Get pool status and metrics"""withself.connection_lock:active_connections=len([connforconninself.connections.values()ifconn['state']==ConnectionState.BUSY])idle_connections=len([connforconninself.connections.values()ifconn['state']==ConnectionState.IDLE])return{"min_connections":self.min_connections,"max_connections":self.max_connections,"active_connections":active_connections,"idle_connections":idle_connections,"total_connections":len(self.connections),"failed_connections":self.metrics.failed_connections,"connection_requests":self.metrics.connection_requests,"avg_wait_time":self.metrics.connection_wait_time/max(self.metrics.connection_requests,1),"last_activity":self.metrics.last_activity.isoformat()ifself.metrics.last_activityelseNone}defclose_pool(self)->None:"""Close all connections in pool"""try:ifself.pool:self.pool.closeall()logger.info("Connection pool closed")exceptExceptionase:logger.error(f"Failed to close connection pool: {e}")classAsyncDatabasePool:"""Async database connection pool"""def__init__(self,connection_string:str,min_connections:int=5,max_connections:int=20):self.connection_string=connection_stringself.min_connections=min_connectionsself.max_connections=max_connectionsself.pool=Noneself.metrics=ConnectionMetrics()self.connection_lock=asyncio.Lock()self.connections={}asyncdefinitialize_pool(self)->None:"""Initialize async connection pool"""try:self.pool=awaitasyncpg.create_pool(self.connection_string,min_size=self.min_connections,max_size=self.max_connections,command_timeout=60)logger.info(f"Async connection pool initialized with {self.min_connections}-{self.max_connections} connections")exceptExceptionase:logger.error(f"Failed to initialize async connection pool: {e}")raiseasyncdefget_connection(self)->asyncpg.Connection:"""Get async connection from pool"""start_time=time.time()try:connection=awaitself.pool.acquire()self.metrics.active_connections+=1self.metrics.connection_requests+=1self.metrics.connection_wait_time+=time.time()-start_timeself.metrics.last_activity=datetime.utcnow()logger.debug(f"Async connection acquired: {id(connection)}")returnconnectionexceptExceptionase:self.metrics.failed_connections+=1logger.error(f"Failed to get async connection: {e}")raiseasyncdefreturn_connection(self,connection:asyncpg.Connection)->None:"""Return async connection to pool"""try:awaitself.pool.release(connection)self.metrics.active_connections-=1self.metrics.idle_connections+=1logger.debug(f"Async connection returned: {id(connection)}")exceptExceptionase:logger.error(f"Failed to return async connection: {e}")asyncdefclose_pool(self)->None:"""Close async connection pool"""try:ifself.pool:awaitself.pool.close()logger.info("Async connection pool closed")exceptExceptionase:logger.error(f"Failed to close async connection pool: {e}")classSQLAlchemyConnectionManager:"""SQLAlchemy connection management"""def__init__(self,connection_string:str,pool_size:int=10,max_overflow:int=20,pool_timeout:int=30):self.connection_string=connection_stringself.engine=Noneself.session_factory=Noneself.pool_size=pool_sizeself.max_overflow=max_overflowself.pool_timeout=pool_timeoutself.initialize_engine()definitialize_engine(self)->None:"""Initialize SQLAlchemy engine with connection pooling"""try:self.engine=create_engine(self.connection_string,poolclass=QueuePool,pool_size=self.pool_size,max_overflow=self.max_overflow,pool_timeout=self.pool_timeout,pool_pre_ping=True,pool_recycle=3600,# Recycle connections every hourecho=False)self.session_factory=sessionmaker(bind=self.engine)logger.info(f"SQLAlchemy engine initialized with pool size {self.pool_size}")exceptExceptionase:logger.error(f"Failed to initialize SQLAlchemy engine: {e}")raise@contextmanagerdefget_session(self):"""Get database session with automatic cleanup"""session=self.session_factory()try:yieldsessionsession.commit()exceptExceptionase:session.rollback()logger.error(f"Database session error: {e}")raisefinally:session.close()defget_engine_status(self)->Dict[str,Any]:"""Get engine and pool status"""pool=self.engine.poolreturn{"pool_size":pool.size(),"checked_in":pool.checkedin(),"checked_out":pool.checkedout(),"overflow":pool.overflow(),"invalid":pool.invalid()}classAsyncSQLAlchemyManager:"""Async SQLAlchemy connection management"""def__init__(self,connection_string:str,pool_size:int=10,max_overflow:int=20):self.connection_string=connection_stringself.engine=Noneself.session_factory=Noneself.pool_size=pool_sizeself.max_overflow=max_overflowself.initialize_engine()definitialize_engine(self)->None:"""Initialize async SQLAlchemy engine"""try:self.engine=create_async_engine(self.connection_string,pool_size=self.pool_size,max_overflow=self.max_overflow,pool_pre_ping=True,pool_recycle=3600,echo=False)self.session_factory=sessionmaker(bind=self.engine,class_=AsyncSession,expire_on_commit=False)logger.info(f"Async SQLAlchemy engine initialized with pool size {self.pool_size}")exceptExceptionase:logger.error(f"Failed to initialize async SQLAlchemy engine: {e}")raise@asynccontextmanagerasyncdefget_session(self):"""Get async database session with automatic cleanup"""session=self.session_factory()try:yieldsessionawaitsession.commit()exceptExceptionase:awaitsession.rollback()logger.error(f"Async database session error: {e}")raisefinally:awaitsession.close()classRedisConnectionManager:"""Redis connection management"""def__init__(self,host:str="localhost",port:int=6379,db:int=0,max_connections:int=20):self.host=hostself.port=portself.db=dbself.max_connections=max_connectionsself.pool=Noneself.initialize_pool()definitialize_pool(self)->None:"""Initialize Redis connection pool"""try:self.pool=RedisConnectionPool(host=self.host,port=self.port,db=self.db,max_connections=self.max_connections,retry_on_timeout=True)logger.info(f"Redis connection pool initialized with {self.max_connections} connections")exceptExceptionase:logger.error(f"Failed to initialize Redis connection pool: {e}")raisedefget_connection(self):"""Get Redis connection from pool"""try:fromredisimportRedisreturnRedis(connection_pool=self.pool)exceptExceptionase:logger.error(f"Failed to get Redis connection: {e}")raisedefget_pool_status(self)->Dict[str,Any]:"""Get Redis pool status"""return{"max_connections":self.max_connections,"created_connections":self.pool.created_connections,"available_connections":self.pool.available_connections,"in_use_connections":self.pool.in_use_connections}# Usage examplesdefexample_connection_pooling():"""Example connection pooling usage"""# PostgreSQL connection poolpg_pool=DatabaseConnectionPool(connection_string="postgresql://user:pass@localhost/db",min_connections=5,max_connections=20)# Get connectionconnection=pg_pool.get_connection()try:cursor=connection.cursor()cursor.execute("SELECT 1")result=cursor.fetchone()print(f"Query result: {result}")finally:pg_pool.return_connection(connection)# Get pool statusstatus=pg_pool.get_pool_status()print(f"Pool status: {status}")# SQLAlchemy connection managersqlalchemy_manager=SQLAlchemyConnectionManager(connection_string="postgresql://user:pass@localhost/db",pool_size=10,max_overflow=20)# Use sessionwithsqlalchemy_manager.get_session()assession:result=session.execute(text("SELECT 1")).fetchone()print(f"SQLAlchemy result: {result}")# Redis connection managerredis_manager=RedisConnectionManager(host="localhost",port=6379,max_connections=20)# Use Redis connectionredis_client=redis_manager.get_connection()redis_client.set("test_key","test_value")value=redis_client.get("test_key")print(f"Redis value: {value}")# Close poolspg_pool.close_pool()
# python/02-transaction-management.py"""Database transaction management patterns and ACID compliance"""fromtypingimportList,Dict,Any,Optional,Union,Tuple,Callablefromdataclassesimportdataclass,asdictfromenumimportEnumimportasyncioimporttimefromcontextlibimportasynccontextmanager,contextmanagerfromdatetimeimportdatetime,timedeltaimportloggingfromfunctoolsimportwrapsimportpsycopg2frompsycopg2.extensionsimportISOLATION_LEVEL_READ_COMMITTED,ISOLATION_LEVEL_REPEATABLE_READfromsqlalchemy.ormimportSessionfromsqlalchemy.excimportSQLAlchemyErrorimportasyncpgfromasyncpgimportConnectionlogger=logging.getLogger(__name__)classTransactionIsolation(Enum):"""Transaction isolation levels"""READ_UNCOMMITTED="READ UNCOMMITTED"READ_COMMITTED="READ COMMITTED"REPEATABLE_READ="REPEATABLE READ"SERIALIZABLE="SERIALIZABLE"classTransactionManager:"""Database transaction manager"""def__init__(self,connection_pool:DatabaseConnectionPool):self.connection_pool=connection_poolself.transaction_metrics={"total_transactions":0,"successful_transactions":0,"failed_transactions":0,"rollback_count":0,"avg_transaction_time":0.0}@contextmanagerdeftransaction(self,isolation_level:TransactionIsolation=TransactionIsolation.READ_COMMITTED):"""Context manager for database transactions"""connection=Nonestart_time=time.time()try:# Get connection from poolconnection=self.connection_pool.get_connection()# Set isolation levelifisolation_level==TransactionIsolation.READ_COMMITTED:connection.set_isolation_level(ISOLATION_LEVEL_READ_COMMITTED)elifisolation_level==TransactionIsolation.REPEATABLE_READ:connection.set_isolation_level(ISOLATION_LEVEL_REPEATABLE_READ)# Start transactionconnection.autocommit=Falseself.transaction_metrics["total_transactions"]+=1yieldconnection# Commit transactionconnection.commit()self.transaction_metrics["successful_transactions"]+=1transaction_time=time.time()-start_timeself._update_avg_transaction_time(transaction_time)logger.debug(f"Transaction committed in {transaction_time:.3f}s")exceptExceptionase:# Rollback transactionifconnection:connection.rollback()self.transaction_metrics["rollback_count"]+=1logger.error(f"Transaction rolled back: {e}")self.transaction_metrics["failed_transactions"]+=1raisefinally:# Return connection to poolifconnection:self.connection_pool.return_connection(connection)def_update_avg_transaction_time(self,transaction_time:float)->None:"""Update average transaction time"""total_transactions=self.transaction_metrics["total_transactions"]current_avg=self.transaction_metrics["avg_transaction_time"]# Calculate new averagenew_avg=((current_avg*(total_transactions-1))+transaction_time)/total_transactionsself.transaction_metrics["avg_transaction_time"]=new_avgdefget_transaction_metrics(self)->Dict[str,Any]:"""Get transaction metrics"""total=self.transaction_metrics["total_transactions"]success_rate=(self.transaction_metrics["successful_transactions"]/total*100)iftotal>0else0return{**self.transaction_metrics,"success_rate":success_rate}classAsyncTransactionManager:"""Async database transaction manager"""def__init__(self,connection_pool:AsyncDatabasePool):self.connection_pool=connection_poolself.transaction_metrics={"total_transactions":0,"successful_transactions":0,"failed_transactions":0,"rollback_count":0,"avg_transaction_time":0.0}@asynccontextmanagerasyncdeftransaction(self,isolation_level:str="READ COMMITTED"):"""Async context manager for database transactions"""connection=Nonestart_time=time.time()try:# Get connection from poolconnection=awaitself.connection_pool.get_connection()# Start transactionasyncwithconnection.transaction(isolation=isolation_level):self.transaction_metrics["total_transactions"]+=1yieldconnection# Transaction will be committed automaticallyself.transaction_metrics["successful_transactions"]+=1transaction_time=time.time()-start_timeself._update_avg_transaction_time(transaction_time)logger.debug(f"Async transaction committed in {transaction_time:.3f}s")exceptExceptionase:# Transaction will be rolled back automaticallyself.transaction_metrics["failed_transactions"]+=1self.transaction_metrics["rollback_count"]+=1logger.error(f"Async transaction rolled back: {e}")raisefinally:# Return connection to poolifconnection:awaitself.connection_pool.return_connection(connection)def_update_avg_transaction_time(self,transaction_time:float)->None:"""Update average transaction time"""total_transactions=self.transaction_metrics["total_transactions"]current_avg=self.transaction_metrics["avg_transaction_time"]new_avg=((current_avg*(total_transactions-1))+transaction_time)/total_transactionsself.transaction_metrics["avg_transaction_time"]=new_avgdefget_transaction_metrics(self)->Dict[str,Any]:"""Get transaction metrics"""total=self.transaction_metrics["total_transactions"]success_rate=(self.transaction_metrics["successful_transactions"]/total*100)iftotal>0else0return{**self.transaction_metrics,"success_rate":success_rate}classSQLAlchemyTransactionManager:"""SQLAlchemy transaction manager"""def__init__(self,session_factory):self.session_factory=session_factoryself.transaction_metrics={"total_transactions":0,"successful_transactions":0,"failed_transactions":0,"rollback_count":0,"avg_transaction_time":0.0}@contextmanagerdeftransaction(self):"""SQLAlchemy transaction context manager"""session=self.session_factory()start_time=time.time()try:self.transaction_metrics["total_transactions"]+=1yieldsession# Commit transactionsession.commit()self.transaction_metrics["successful_transactions"]+=1transaction_time=time.time()-start_timeself._update_avg_transaction_time(transaction_time)logger.debug(f"SQLAlchemy transaction committed in {transaction_time:.3f}s")exceptSQLAlchemyErrorase:# Rollback transactionsession.rollback()self.transaction_metrics["rollback_count"]+=1self.transaction_metrics["failed_transactions"]+=1logger.error(f"SQLAlchemy transaction rolled back: {e}")raisefinally:session.close()def_update_avg_transaction_time(self,transaction_time:float)->None:"""Update average transaction time"""total_transactions=self.transaction_metrics["total_transactions"]current_avg=self.transaction_metrics["avg_transaction_time"]new_avg=((current_avg*(total_transactions-1))+transaction_time)/total_transactionsself.transaction_metrics["avg_transaction_time"]=new_avgdefget_transaction_metrics(self)->Dict[str,Any]:"""Get transaction metrics"""total=self.transaction_metrics["total_transactions"]success_rate=(self.transaction_metrics["successful_transactions"]/total*100)iftotal>0else0return{**self.transaction_metrics,"success_rate":success_rate}deftransactional(isolation_level:TransactionIsolation=TransactionIsolation.READ_COMMITTED):"""Decorator for transactional methods"""defdecorator(func:Callable)->Callable:@wraps(func)defwrapper(*args,**kwargs):# This would be implemented with actual transaction manager# For now, just execute the functionreturnfunc(*args,**kwargs)returnwrapperreturndecoratorclassDatabaseRepository:"""Repository pattern for database operations"""def__init__(self,transaction_manager:TransactionManager):self.transaction_manager=transaction_managerdefcreate_user(self,user_data:Dict[str,Any])->int:"""Create user with transaction"""withself.transaction_manager.transaction()asconn:cursor=conn.cursor()# Insert usercursor.execute(""" INSERT INTO users (name, email, created_at) VALUES (%s, %s, %s) RETURNING id """,(user_data['name'],user_data['email'],datetime.utcnow()))user_id=cursor.fetchone()[0]# Insert user profilecursor.execute(""" INSERT INTO user_profiles (user_id, bio, avatar_url) VALUES (%s, %s, %s) """,(user_id,user_data.get('bio',''),user_data.get('avatar_url','')))returnuser_iddefget_user(self,user_id:int)->Optional[Dict[str,Any]]:"""Get user by ID"""withself.transaction_manager.transaction()asconn:cursor=conn.cursor()cursor.execute(""" SELECT u.id, u.name, u.email, u.created_at, p.bio, p.avatar_url FROM users u LEFT JOIN user_profiles p ON u.id = p.user_id WHERE u.id = %s """,(user_id,))result=cursor.fetchone()ifresult:return{'id':result[0],'name':result[1],'email':result[2],'created_at':result[3],'bio':result[4],'avatar_url':result[5]}returnNonedefupdate_user(self,user_id:int,user_data:Dict[str,Any])->bool:"""Update user with transaction"""withself.transaction_manager.transaction()asconn:cursor=conn.cursor()# Update usercursor.execute(""" UPDATE users SET name = %s, email = %s, updated_at = %s WHERE id = %s """,(user_data['name'],user_data['email'],datetime.utcnow(),user_id))# Update user profilecursor.execute(""" UPDATE user_profiles SET bio = %s, avatar_url = %s WHERE user_id = %s """,(user_data.get('bio',''),user_data.get('avatar_url',''),user_id))returncursor.rowcount>0defdelete_user(self,user_id:int)->bool:"""Delete user with transaction"""withself.transaction_manager.transaction()asconn:cursor=conn.cursor()# Delete user profile first (foreign key constraint)cursor.execute("DELETE FROM user_profiles WHERE user_id = %s",(user_id,))# Delete usercursor.execute("DELETE FROM users WHERE id = %s",(user_id,))returncursor.rowcount>0classDatabaseMigrationManager:"""Database migration management"""def__init__(self,connection_pool:DatabaseConnectionPool):self.connection_pool=connection_poolself.migrations=[]self.initialize_migrations_table()definitialize_migrations_table(self)->None:"""Initialize migrations tracking table"""withself.connection_pool.get_connection()asconn:cursor=conn.cursor()cursor.execute(""" CREATE TABLE IF NOT EXISTS schema_migrations ( id SERIAL PRIMARY KEY, version VARCHAR(255) UNIQUE NOT NULL, applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """)conn.commit()defadd_migration(self,version:str,sql:str)->None:"""Add migration"""self.migrations.append({'version':version,'sql':sql,'applied':False})defrun_migrations(self)->List[str]:"""Run pending migrations"""applied_migrations=[]withself.connection_pool.get_connection()asconn:cursor=conn.cursor()# Get applied migrationscursor.execute("SELECT version FROM schema_migrations")applied_versions={row[0]forrowincursor.fetchall()}# Run pending migrationsformigrationinself.migrations:ifmigration['version']notinapplied_versions:try:cursor.execute(migration['sql'])cursor.execute("INSERT INTO schema_migrations (version) VALUES (%s)",(migration['version'],))conn.commit()applied_migrations.append(migration['version'])logger.info(f"Applied migration: {migration['version']}")exceptExceptionase:conn.rollback()logger.error(f"Failed to apply migration {migration['version']}: {e}")raisereturnapplied_migrations# Usage examplesdefexample_transaction_management():"""Example transaction management usage"""# Create connection poolconnection_pool=DatabaseConnectionPool("postgresql://user:pass@localhost/db")# Create transaction managertransaction_manager=TransactionManager(connection_pool)# Use transactionwithtransaction_manager.transaction()asconn:cursor=conn.cursor()cursor.execute("INSERT INTO users (name, email) VALUES (%s, %s)",("John","john@example.com"))cursor.execute("INSERT INTO user_profiles (user_id, bio) VALUES (%s, %s)",(1,"Software Developer"))# Get transaction metricsmetrics=transaction_manager.get_transaction_metrics()print(f"Transaction metrics: {metrics}")# Repository patternrepository=DatabaseRepository(transaction_manager)# Create useruser_data={'name':'Jane Doe','email':'jane@example.com','bio':'Data Scientist','avatar_url':'https://example.com/avatar.jpg'}user_id=repository.create_user(user_data)print(f"Created user with ID: {user_id}")# Get useruser=repository.get_user(user_id)print(f"Retrieved user: {user}")# Database migrationsmigration_manager=DatabaseMigrationManager(connection_pool)# Add migrationmigration_manager.add_migration("001_create_users_table",""" CREATE TABLE users ( id SERIAL PRIMARY KEY, name VARCHAR(255) NOT NULL, email VARCHAR(255) UNIQUE NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) """)# Run migrationsapplied_migrations=migration_manager.run_migrations()print(f"Applied migrations: {applied_migrations}")# Close connection poolconnection_pool.close_pool()