# python/01-memory-architecture.py"""Understanding Python's memory model and architecture"""importsysimportgcimportweakreffromtypingimportList,Dict,Any,Optional,CallableimporttracemallocimportpsutilimportosclassMemoryAnalyzer:"""Advanced memory analysis and monitoring"""def__init__(self):self.snapshots=[]self.memory_traces=[]self.reference_tracker={}defget_memory_info(self)->Dict[str,Any]:"""Get comprehensive memory information"""process=psutil.Process()memory_info=process.memory_info()return{"rss":memory_info.rss,# Resident Set Size"vms":memory_info.vms,# Virtual Memory Size"percent":process.memory_percent(),"available":psutil.virtual_memory().available,"total":psutil.virtual_memory().total,"python_objects":len(gc.get_objects()),"gc_counts":gc.get_count()}defstart_memory_tracing(self):"""Start memory tracing"""tracemalloc.start()self.memory_traces.append(tracemalloc.get_traced_memory())defstop_memory_tracing(self)->Dict[str,Any]:"""Stop memory tracing and get stats"""current,peak=tracemalloc.get_traced_memory()tracemalloc.stop()return{"current_memory":current,"peak_memory":peak,"current_mb":current/1024/1024,"peak_mb":peak/1024/1024}deftake_memory_snapshot(self)->Dict[str,Any]:"""Take memory snapshot"""snapshot=self.get_memory_info()self.snapshots.append(snapshot)returnsnapshotdefcompare_snapshots(self,snapshot1:int,snapshot2:int)->Dict[str,Any]:"""Compare two memory snapshots"""ifsnapshot1>=len(self.snapshots)orsnapshot2>=len(self.snapshots):raiseValueError("Invalid snapshot indices")snap1=self.snapshots[snapshot1]snap2=self.snapshots[snapshot2]return{"rss_diff":snap2["rss"]-snap1["rss"],"vms_diff":snap2["vms"]-snap1["vms"],"percent_diff":snap2["percent"]-snap1["percent"],"objects_diff":snap2["python_objects"]-snap1["python_objects"]}deffind_memory_leaks(self)->Dict[str,Any]:"""Find potential memory leaks"""# Force garbage collectiongc.collect()# Get object counts by typeobject_counts={}forobjingc.get_objects():obj_type=type(obj).__name__object_counts[obj_type]=object_counts.get(obj_type,0)+1# Find objects with high counts (potential leaks)potential_leaks={obj_type:countforobj_type,countinobject_counts.items()ifcount>1000# Threshold for potential leaks}return{"object_counts":object_counts,"potential_leaks":potential_leaks,"gc_counts":gc.get_count(),"total_objects":len(gc.get_objects())}classReferenceTracker:"""Track object references and prevent leaks"""def__init__(self):self.tracked_objects={}self.weak_refs={}self.circular_refs=[]deftrack_object(self,obj:Any,name:str)->str:"""Track an object and return tracking ID"""obj_id=id(obj)self.tracked_objects[obj_id]={"object":obj,"name":name,"type":type(obj).__name__,"ref_count":sys.getrefcount(obj)}returnstr(obj_id)defcreate_weak_reference(self,obj:Any,callback:Optional[Callable]=None)->weakref.ref:"""Create weak reference to object"""weak_ref=weakref.ref(obj,callback)self.weak_refs[id(obj)]=weak_refreturnweak_refdefcheck_circular_references(self)->List[Dict[str,Any]]:"""Check for circular references"""circular_refs=[]forobjingc.get_objects():ifgc.is_tracked(obj):referrers=gc.get_referrers(obj)forreferrerinreferrers:ifgc.is_tracked(referrer):# Check if they reference each otherifobjingc.get_referrers(referrer):circular_refs.append({"obj1":type(obj).__name__,"obj2":type(referrer).__name__,"obj1_id":id(obj),"obj2_id":id(referrer)})self.circular_refs=circular_refsreturncircular_refsdefget_reference_info(self,obj_id:str)->Optional[Dict[str,Any]]:"""Get reference information for tracked object"""obj_id_int=int(obj_id)ifobj_id_intinself.tracked_objects:obj_info=self.tracked_objects[obj_id_int]return{"name":obj_info["name"],"type":obj_info["type"],"ref_count":sys.getrefcount(obj_info["object"]),"is_tracked":gc.is_tracked(obj_info["object"])}returnNonedefcleanup_tracked_objects(self):"""Cleanup tracked objects"""self.tracked_objects.clear()self.weak_refs.clear()self.circular_refs.clear()# Usage examplesdefexample_memory_analysis():"""Example memory analysis usage"""analyzer=MemoryAnalyzer()# Take initial snapshotinitial_snapshot=analyzer.take_memory_snapshot()print(f"Initial memory: {initial_snapshot['rss']/1024/1024:.2f} MB")# Create some objectsdata=[iforiinrange(10000)]# Take second snapshotsecond_snapshot=analyzer.take_memory_snapshot()print(f"After creating data: {second_snapshot['rss']/1024/1024:.2f} MB")# Compare snapshotscomparison=analyzer.compare_snapshots(0,1)print(f"Memory difference: {comparison['rss_diff']/1024/1024:.2f} MB")# Check for leaksleak_info=analyzer.find_memory_leaks()print(f"Potential leaks: {leak_info['potential_leaks']}")defexample_reference_tracking():"""Example reference tracking usage"""tracker=ReferenceTracker()# Create and track objectsobj1=[1,2,3]obj2={"key":"value"}id1=tracker.track_object(obj1,"list_object")id2=tracker.track_object(obj2,"dict_object")# Get reference infoinfo1=tracker.get_reference_info(id1)info2=tracker.get_reference_info(id2)print(f"Object 1 ref count: {info1['ref_count']}")print(f"Object 2 ref count: {info2['ref_count']}")# Check for circular referencescircular_refs=tracker.check_circular_references()print(f"Circular references found: {len(circular_refs)}")
# python/02-garbage-collection.py"""Advanced garbage collection management and optimization"""importgcimportsysimporttimefromtypingimportList,Dict,Any,Optional,CallableimportweakreffromcontextlibimportcontextmanagerclassGarbageCollectionManager:"""Advanced garbage collection management"""def__init__(self):self.gc_stats=[]self.collection_times=[]self.thresholds=gc.get_threshold()defget_gc_stats(self)->Dict[str,Any]:"""Get comprehensive GC statistics"""counts=gc.get_count()stats={"generation_0":counts[0],"generation_1":counts[1],"generation_2":counts[2],"thresholds":self.thresholds,"total_objects":len(gc.get_objects()),"tracked_objects":len([objforobjingc.get_objects()ifgc.is_tracked(obj)])}self.gc_stats.append(stats)returnstatsdefforce_collection(self)->Dict[str,Any]:"""Force garbage collection and measure performance"""start_time=time.time()# Get initial statsinitial_stats=self.get_gc_stats()# Force collectioncollected=gc.collect()# Get final statsfinal_stats=self.get_gc_stats()collection_time=time.time()-start_timeself.collection_times.append(collection_time)return{"collected_objects":collected,"collection_time":collection_time,"initial_objects":initial_stats["total_objects"],"final_objects":final_stats["total_objects"],"objects_freed":initial_stats["total_objects"]-final_stats["total_objects"]}defoptimize_gc_thresholds(self,target_objects:int=10000):"""Optimize GC thresholds based on object count"""current_objects=len(gc.get_objects())ifcurrent_objects>target_objects:# Increase thresholds to reduce GC frequencynew_thresholds=(self.thresholds[0]*2,self.thresholds[1]*2,self.thresholds[2]*2)gc.set_threshold(*new_thresholds)self.thresholds=new_thresholdsreturnTruereturnFalsedefdisable_gc(self):"""Disable garbage collection"""gc.disable()defenable_gc(self):"""Enable garbage collection"""gc.enable()@contextmanagerdefgc_context(self,enabled:bool=True):"""Context manager for GC control"""original_state=gc.isenabled()ifenabled:gc.enable()else:gc.disable()try:yieldfinally:iforiginal_state:gc.enable()else:gc.disable()classMemoryEfficientDataStructures:"""Memory-efficient data structure implementations"""@staticmethoddefmemory_efficient_list(size:int)->List[int]:"""Create memory-efficient list"""# Use generator for large listsreturn[iforiinrange(size)]@staticmethoddefmemory_efficient_dict(keys:List[str],values:List[Any])->Dict[str,Any]:"""Create memory-efficient dictionary"""# Use dict comprehensionreturn{k:vfork,vinzip(keys,values)}@staticmethoddefmemory_efficient_set(items:List[Any])->set:"""Create memory-efficient set"""returnset(items)@staticmethoddefmemory_efficient_string_concatenation(strings:List[str])->str:"""Memory-efficient string concatenation"""# Use join instead of +=return''.join(strings)@staticmethoddefmemory_efficient_file_processing(file_path:str,chunk_size:int=8192):"""Memory-efficient file processing"""withopen(file_path,'r')asfile:whileTrue:chunk=file.read(chunk_size)ifnotchunk:breakyieldchunkclassWeakReferenceManager:"""Manage weak references to prevent memory leaks"""def__init__(self):self.weak_refs={}self.callbacks={}defcreate_weak_reference(self,obj:Any,name:str,callback:Optional[Callable]=None)->weakref.ref:"""Create weak reference with callback"""defcleanup_callback(weak_ref):ifnameinself.weak_refs:delself.weak_refs[name]ifnameinself.callbacks:delself.callbacks[name]ifcallback:callback(weak_ref)weak_ref=weakref.ref(obj,cleanup_callback)self.weak_refs[name]=weak_refifcallback:self.callbacks[name]=callbackreturnweak_refdefget_weak_reference(self,name:str)->Optional[Any]:"""Get object from weak reference"""ifnameinself.weak_refs:returnself.weak_refs[name]()returnNonedefcleanup_weak_references(self):"""Cleanup all weak references"""self.weak_refs.clear()self.callbacks.clear()# Usage examplesdefexample_gc_management():"""Example garbage collection management"""gc_manager=GarbageCollectionManager()# Get initial GC statsinitial_stats=gc_manager.get_gc_stats()print(f"Initial objects: {initial_stats['total_objects']}")# Create some objectsdata=[iforiinrange(10000)]# Force collectioncollection_result=gc_manager.force_collection()print(f"Collected {collection_result['collected_objects']} objects")print(f"Collection time: {collection_result['collection_time']:.4f}s")# Optimize thresholdsoptimized=gc_manager.optimize_gc_thresholds()print(f"GC thresholds optimized: {optimized}")defexample_memory_efficient_structures():"""Example memory-efficient data structures"""# Memory-efficient listlarge_list=MemoryEfficientDataStructures.memory_efficient_list(10000)print(f"List size: {sys.getsizeof(large_list)} bytes")# Memory-efficient string concatenationstrings=[f"string_{i}"foriinrange(1000)]result=MemoryEfficientDataStructures.memory_efficient_string_concatenation(strings)print(f"Concatenated string length: {len(result)}")# Memory-efficient file processing# This would process a file in chunks# for chunk in MemoryEfficientDataStructures.memory_efficient_file_processing("large_file.txt"):# process_chunk(chunk)
# python/03-memory-optimization.py"""Memory optimization patterns including object pooling and caching"""importgcimportsysimporttimefromtypingimportList,Dict,Any,Optional,Type,CallablefromcollectionsimportdequeimportthreadingclassObjectPool:"""Generic object pool for memory optimization"""def__init__(self,factory:Callable,max_size:int=100,reset_func:Optional[Callable]=None):self.factory=factoryself.max_size=max_sizeself.reset_func=reset_funcself.pool=deque()self.lock=threading.Lock()self.created_count=0self.reused_count=0defget_object(self)->Any:"""Get object from pool"""withself.lock:ifself.pool:obj=self.pool.popleft()self.reused_count+=1else:obj=self.factory()self.created_count+=1# Reset object if reset function providedifself.reset_func:self.reset_func(obj)returnobjdefreturn_object(self,obj:Any):"""Return object to pool"""withself.lock:iflen(self.pool)<self.max_size:self.pool.append(obj)defget_stats(self)->Dict[str,Any]:"""Get pool statistics"""withself.lock:return{"pool_size":len(self.pool),"created_count":self.created_count,"reused_count":self.reused_count,"total_objects":self.created_count+self.reused_count}defclear_pool(self):"""Clear object pool"""withself.lock:self.pool.clear()classMemoryCache:"""Memory-efficient cache with size limits"""def__init__(self,max_size:int=1000,max_memory_mb:int=100):self.max_size=max_sizeself.max_memory_bytes=max_memory_mb*1024*1024self.cache={}self.access_order=deque()self.current_memory=0self.lock=threading.Lock()defget(self,key:str)->Optional[Any]:"""Get value from cache"""withself.lock:ifkeyinself.cache:# Update access orderifkeyinself.access_order:self.access_order.remove(key)self.access_order.append(key)returnself.cache[key]returnNonedefput(self,key:str,value:Any)->None:"""Put value in cache"""withself.lock:# Calculate memory usagevalue_size=sys.getsizeof(value)# Remove items if cache is fullwhile(len(self.cache)>=self.max_sizeorself.current_memory+value_size>self.max_memory_bytes):ifnotself.access_order:breakoldest_key=self.access_order.popleft()ifoldest_keyinself.cache:old_value=self.cache[oldest_key]self.current_memory-=sys.getsizeof(old_value)delself.cache[oldest_key]# Add new itemself.cache[key]=valueself.current_memory+=value_sizeself.access_order.append(key)defclear(self)->None:"""Clear cache"""withself.lock:self.cache.clear()self.access_order.clear()self.current_memory=0defget_stats(self)->Dict[str,Any]:"""Get cache statistics"""withself.lock:return{"size":len(self.cache),"memory_usage":self.current_memory,"memory_usage_mb":self.current_memory/1024/1024,"max_size":self.max_size,"max_memory_mb":self.max_memory_bytes/1024/1024}classMemoryMonitor:"""Monitor memory usage and provide alerts"""def__init__(self,warning_threshold:float=80.0,critical_threshold:float=95.0):self.warning_threshold=warning_thresholdself.critical_threshold=critical_thresholdself.monitoring=Falseself.monitor_thread=Noneself.alerts=[]defstart_monitoring(self,interval:float=1.0):"""Start memory monitoring"""ifself.monitoring:returnself.monitoring=Trueself.monitor_thread=threading.Thread(target=self._monitor_loop,args=(interval,),daemon=True)self.monitor_thread.start()defstop_monitoring(self):"""Stop memory monitoring"""self.monitoring=Falseifself.monitor_thread:self.monitor_thread.join()def_monitor_loop(self,interval:float):"""Memory monitoring loop"""whileself.monitoring:try:memory_percent=psutil.virtual_memory().percentifmemory_percent>=self.critical_threshold:self.alerts.append({"level":"critical","memory_percent":memory_percent,"timestamp":time.time()})elifmemory_percent>=self.warning_threshold:self.alerts.append({"level":"warning","memory_percent":memory_percent,"timestamp":time.time()})time.sleep(interval)exceptExceptionase:print(f"Memory monitoring error: {e}")time.sleep(interval)defget_alerts(self)->List[Dict[str,Any]]:"""Get memory alerts"""returnself.alerts.copy()defclear_alerts(self):"""Clear memory alerts"""self.alerts.clear()# Usage examplesdefexample_object_pooling():"""Example object pooling usage"""# Create object pooldefcreate_expensive_object():return{"data":[iforiinrange(1000)],"timestamp":time.time()}defreset_object(obj):obj["data"].clear()obj["timestamp"]=time.time()pool=ObjectPool(create_expensive_object,max_size=10,reset_func=reset_object)# Use objects from poolobj1=pool.get_object()obj2=pool.get_object()# Return objects to poolpool.return_object(obj1)pool.return_object(obj2)# Get pool statsstats=pool.get_stats()print(f"Pool stats: {stats}")defexample_memory_cache():"""Example memory cache usage"""cache=MemoryCache(max_size=100,max_memory_mb=10)# Add items to cacheforiinrange(50):cache.put(f"key_{i}",f"value_{i}"*100)# Get cache statsstats=cache.get_stats()print(f"Cache stats: {stats}")# Get item from cachevalue=cache.get("key_10")print(f"Retrieved value: {value}")defexample_memory_monitoring():"""Example memory monitoring usage"""monitor=MemoryMonitor(warning_threshold=70.0,critical_threshold=90.0)# Start monitoringmonitor.start_monitoring(interval=0.5)# Simulate memory usagedata=[]foriinrange(1000):data.append([jforjinrange(1000)])time.sleep(0.01)# Check alertsalerts=monitor.get_alerts()print(f"Memory alerts: {len(alerts)}")# Stop monitoringmonitor.stop_monitoring()
# 1. Monitor memory usagefrompython.memory_managementimportMemoryAnalyzeranalyzer=MemoryAnalyzer()memory_info=analyzer.get_memory_info()print(f"Memory usage: {memory_info['rss']/1024/1024:.2f} MB")# 2. Use object poolingfrompython.memory_optimizationimportObjectPoolpool=ObjectPool(create_expensive_object,max_size=10)obj=pool.get_object()# Use objectpool.return_object(obj)# 3. Monitor for leaksfrompython.memory_leak_preventionimportmemory_contextwithmemory_context(monitor_leaks=True)asdetector:# Your code herepass# 4. Force garbage collectionimportgcgc.collect()