mirror of
https://github.com/andwati/student-retention.git
synced 2026-10-28 21:22:15 +00:00
yolo
This commit is contained in:
@@ -1,92 +0,0 @@
|
||||
"""
|
||||
System Overview Diagram
|
||||
High-level view of the entire Student Success Prediction system
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.onprem.client import Users
|
||||
from diagrams.generic.storage import Storage
|
||||
from diagrams.programming.language import Python
|
||||
from diagrams.programming.framework import Django
|
||||
from diagrams.onprem.analytics import Tableau
|
||||
from diagrams.onprem.inmemory import Redis
|
||||
from diagrams.aws.ml import SagemakerModel
|
||||
from diagrams.generic.compute import Rack
|
||||
from diagrams.onprem.monitoring import Grafana
|
||||
from diagrams.onprem.network import Nginx
|
||||
|
||||
with Diagram("Student Success Prediction - System Overview",
|
||||
filename="diagrams/00_system_overview",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# System Title and Context
|
||||
title = Storage("Student Success Prediction System\n• 93.84% Accuracy Random Forest Model\n• 36 Features, 3 Outcomes\n• 4,424 Student Dataset")
|
||||
|
||||
# Phase 1: Data Analysis & Model Development
|
||||
with Cluster("Phase 1: Data Analysis & Model Development"):
|
||||
raw_data = Storage("Raw Dataset\ndata.csv")
|
||||
eda_process = Tableau("Exploratory Data Analysis\n• Data quality assessment\n• Statistical analysis\n• Feature insights")
|
||||
ml_pipeline = Python("ML Pipeline\n• Feature engineering\n• Model training\n• Performance validation")
|
||||
model_artifacts = SagemakerModel("Model Artifacts\n• Random Forest (93.84%)\n• Preprocessing info\n• Feature mappings")
|
||||
|
||||
# Phase 2: Application Development
|
||||
with Cluster("Phase 2: Application Development"):
|
||||
modular_architecture = Rack("Modular Architecture\n• Component separation\n• Configuration management\n• Utility services")
|
||||
|
||||
auth_system = Redis("Authentication System\n• Role-based access\n• Session management\n• Security controls")
|
||||
|
||||
ui_components = Django("UI Components\n• Rose Pine theme\n• Interactive forms\n• Responsive design")
|
||||
|
||||
streamlit_app = Django("Streamlit Application\n• Main dashboard\n• User interface\n• API integration")
|
||||
|
||||
# Phase 3: Production Deployment
|
||||
with Cluster("Phase 3: Production Deployment"):
|
||||
deployment_options = Rack("Deployment Options\n• Streamlit Cloud\n• Docker containers\n• Traditional servers")
|
||||
|
||||
monitoring = Grafana("Monitoring & Analytics\n• Performance metrics\n• User analytics\n• System health")
|
||||
|
||||
security = Redis("Security & Compliance\n• Data protection\n• Access control\n• Audit logging")
|
||||
|
||||
# End Users and Outcomes
|
||||
with Cluster("Stakeholders & Users"):
|
||||
students = Users("Students\n• Academic planning\n• Risk assessment\n• Personal insights")
|
||||
educators = Users("Educators\n• Student analytics\n• Intervention planning\n• Performance tracking")
|
||||
administrators = Users("Administrators\n• System management\n• User oversight\n• Strategic planning")
|
||||
|
||||
# Global Internet Access
|
||||
internet = Nginx("Global Access\nSecure HTTPS")
|
||||
|
||||
# System Flow
|
||||
title >> raw_data >> eda_process >> ml_pipeline >> model_artifacts
|
||||
|
||||
model_artifacts >> modular_architecture
|
||||
modular_architecture >> auth_system
|
||||
modular_architecture >> ui_components
|
||||
auth_system >> streamlit_app
|
||||
ui_components >> streamlit_app
|
||||
|
||||
streamlit_app >> deployment_options
|
||||
deployment_options >> monitoring
|
||||
deployment_options >> security
|
||||
|
||||
internet >> streamlit_app
|
||||
streamlit_app >> students
|
||||
streamlit_app >> educators
|
||||
streamlit_app >> administrators
|
||||
|
||||
# Key Metrics and Features
|
||||
with Cluster("Key System Metrics"):
|
||||
accuracy_metric = Storage("Model Performance\n• 93.84% Test Accuracy\n• Balanced class performance\n• Robust generalization")
|
||||
|
||||
feature_count = Storage("Feature Engineering\n• 36 input features\n• 105+ encoded features\n• Comprehensive preprocessing")
|
||||
|
||||
user_experience = Storage("User Experience\n• 3 user roles\n• Intuitive interface\n• Real-time predictions")
|
||||
|
||||
scalability = Storage("Scalability\n• Cloud deployment\n• Modular architecture\n• Production-ready")
|
||||
|
||||
# Connect metrics to main system
|
||||
streamlit_app >> accuracy_metric
|
||||
ml_pipeline >> feature_count
|
||||
ui_components >> user_experience
|
||||
deployment_options >> scalability
|
||||
@@ -1,63 +0,0 @@
|
||||
"""
|
||||
Data Flow Architecture Diagram
|
||||
Shows the complete data pipeline from raw data to model predictions
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.onprem.database import PostgreSQL
|
||||
from diagrams.onprem.inmemory import Redis
|
||||
from diagrams.programming.language import Python
|
||||
from diagrams.onprem.analytics import Tableau
|
||||
from diagrams.onprem.client import Users
|
||||
from diagrams.onprem.workflow import Airflow
|
||||
from diagrams.generic.storage import Storage
|
||||
from diagrams.generic.compute import Rack
|
||||
from diagrams.aws.ml import SagemakerModel
|
||||
from diagrams.programming.framework import Django
|
||||
from diagrams.onprem.monitoring import Grafana
|
||||
|
||||
with Diagram("Student Success Prediction - Data Flow Architecture",
|
||||
filename="diagrams/01_data_flow_architecture",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# Data Sources
|
||||
with Cluster("Data Sources"):
|
||||
raw_data = Storage("Raw Dataset\n(data.csv)\n4,424 students\n37 features")
|
||||
|
||||
# Data Processing Pipeline
|
||||
with Cluster("Data Processing Pipeline"):
|
||||
eda_notebook = Python("Exploratory Data Analysis\n(predictive_analysis.ipynb)")
|
||||
data_cleaning = Rack("Data Cleaning\n• Remove apostrophes\n• Fix typos\n• Handle duplicates")
|
||||
feature_eng = Rack("Feature Engineering\n• One-hot encoding\n• Label encoding\n• Scaling")
|
||||
|
||||
# Model Development
|
||||
with Cluster("Model Development"):
|
||||
model_training = Python("Model Training\n(train_and_save_model.py)")
|
||||
model_validation = Grafana("Model Validation\n• Cross-validation\n• Performance metrics")
|
||||
model_storage = Storage("Model Artifacts\n• random_forest_model.pkl\n• preprocessing_info.json")
|
||||
|
||||
# Production System
|
||||
with Cluster("Production Dashboard"):
|
||||
auth_system = Redis("Authentication\n• Session management\n• Role-based access")
|
||||
streamlit_app = Django("Streamlit App\n(app.py)")
|
||||
|
||||
with Cluster("Modular Components"):
|
||||
theme_config = Python("Theme Config\n• Rose Pine CSS\n• UI constants")
|
||||
form_components = Python("Form Components\n• Input validation\n• User interface")
|
||||
model_utils = Python("Model Utils\n• Preprocessing\n• Inference")
|
||||
|
||||
# End Users
|
||||
users = Users("End Users\n• Students\n• Educators\n• Administrators")
|
||||
|
||||
# Data Flow
|
||||
raw_data >> eda_notebook >> data_cleaning >> feature_eng
|
||||
feature_eng >> model_training >> model_validation >> model_storage
|
||||
|
||||
model_storage >> model_utils
|
||||
auth_system >> streamlit_app
|
||||
theme_config >> streamlit_app
|
||||
form_components >> streamlit_app
|
||||
model_utils >> streamlit_app
|
||||
|
||||
streamlit_app >> users
|
||||
@@ -1,50 +0,0 @@
|
||||
"""
|
||||
EDA Process Flow Diagram
|
||||
Shows the exploratory data analysis workflow and key insights
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.generic.storage import Storage
|
||||
from diagrams.generic.compute import Rack
|
||||
from diagrams.programming.language import Python
|
||||
from diagrams.onprem.analytics import Tableau
|
||||
from diagrams.onprem.monitoring import Grafana
|
||||
|
||||
with Diagram("Exploratory Data Analysis Process Flow",
|
||||
filename="diagrams/02_eda_process_flow",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# Data Input
|
||||
raw_dataset = Storage("Raw Dataset\ndata.csv\n4,424 rows × 37 columns")
|
||||
|
||||
# EDA Steps
|
||||
with Cluster("Data Quality Assessment"):
|
||||
data_info = Rack("Data Info\n• No missing values\n• No duplicates\n• Mixed data types")
|
||||
column_analysis = Rack("Column Analysis\n• 36 predictors\n• 1 target variable\n• Column name cleanup")
|
||||
|
||||
with Cluster("Statistical Analysis"):
|
||||
descriptive_stats = Grafana("Descriptive Statistics\n• Central tendencies\n• Distributions\n• Outlier detection")
|
||||
correlation_analysis = Grafana("Correlation Analysis\n• Feature relationships\n• Multicollinearity check")
|
||||
|
||||
with Cluster("Visualization & Insights"):
|
||||
target_distribution = Tableau("Target Distribution\n• Graduate: 50%\n• Dropout: 32%\n• Enrolled: 17%")
|
||||
feature_plots = Tableau("Feature Analysis\n• Histograms\n• Box plots\n• Categorical counts")
|
||||
pattern_discovery = Tableau("Pattern Discovery\n• Key predictors\n• Risk factors\n• Success indicators")
|
||||
|
||||
# Key Findings
|
||||
with Cluster("Key Findings"):
|
||||
class_imbalance = Rack("Class Imbalance\nIdentified for modeling")
|
||||
high_cardinality = Rack("High Cardinality\nCourse (51 values)\nApplication order (9 values)")
|
||||
data_quality = Rack("Data Quality\nClean dataset\nReady for modeling")
|
||||
|
||||
# Data Flow
|
||||
raw_dataset >> data_info >> descriptive_stats
|
||||
raw_dataset >> column_analysis >> correlation_analysis
|
||||
descriptive_stats >> target_distribution
|
||||
correlation_analysis >> feature_plots
|
||||
target_distribution >> pattern_discovery
|
||||
|
||||
pattern_discovery >> class_imbalance
|
||||
pattern_discovery >> high_cardinality
|
||||
pattern_discovery >> data_quality
|
||||
@@ -1,63 +0,0 @@
|
||||
"""
|
||||
Machine Learning Pipeline Architecture
|
||||
Shows the complete ML workflow from preprocessing to model deployment
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.generic.storage import Storage
|
||||
from diagrams.generic.compute import Rack
|
||||
from diagrams.programming.language import Python
|
||||
from diagrams.aws.ml import SagemakerModel
|
||||
from diagrams.onprem.monitoring import Grafana
|
||||
from diagrams.generic.database import SQL
|
||||
|
||||
with Diagram("Machine Learning Pipeline Architecture",
|
||||
filename="diagrams/03_ml_pipeline_architecture",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# Input Data
|
||||
clean_data = Storage("Cleaned Dataset\n• 4,424 samples\n• 36 features\n• 3 target classes")
|
||||
|
||||
# Preprocessing Pipeline
|
||||
with Cluster("Data Preprocessing"):
|
||||
feature_encoding = Rack("Feature Encoding\n• Label encoding (target)\n• One-hot encoding (categorical)")
|
||||
data_splitting = Rack("Data Splitting\n• Train: 75%\n• Test: 25%\n• Stratified split")
|
||||
feature_alignment = Rack("Feature Alignment\n• Consistent column order\n• Missing feature handling")
|
||||
|
||||
# Model Training
|
||||
with Cluster("Model Training & Validation"):
|
||||
model_training = Python("Random Forest Training\n• n_estimators=100\n• random_state=42")
|
||||
cross_validation = Grafana("Cross Validation\n• 5-fold CV\n• Performance metrics")
|
||||
hyperparameter_tuning = Grafana("Hyperparameter Tuning\n• Grid search\n• Best parameters")
|
||||
|
||||
# Model Evaluation
|
||||
with Cluster("Model Evaluation"):
|
||||
performance_metrics = Grafana("Performance Metrics\n• Accuracy: 93.84%\n• Precision/Recall\n• F1-Score")
|
||||
confusion_matrix = Grafana("Confusion Matrix\n• Class-wise performance\n• Error analysis")
|
||||
feature_importance = Grafana("Feature Importance\n• Top predictors\n• Impact analysis")
|
||||
|
||||
# Model Persistence
|
||||
with Cluster("Model Artifacts"):
|
||||
model_pickle = Storage("Model Pickle\nrandom_forest_model.pkl")
|
||||
preprocessing_info = SQL("Preprocessing Info\npreprocessing_info.json\n• Feature names\n• Encodings\n• Mappings")
|
||||
|
||||
# Model Validation Results
|
||||
with Cluster("Validation Results"):
|
||||
accuracy_report = Rack("Accuracy Report\n• Training: 100%\n• Testing: 93.84%\n• Generalization: Good")
|
||||
class_performance = Rack("Class Performance\n• Graduate: 95% F1\n• Dropout: 92% F1\n• Enrolled: 88% F1")
|
||||
|
||||
# Data Flow
|
||||
clean_data >> feature_encoding >> data_splitting
|
||||
data_splitting >> model_training >> cross_validation
|
||||
cross_validation >> hyperparameter_tuning >> performance_metrics
|
||||
|
||||
performance_metrics >> confusion_matrix
|
||||
performance_metrics >> feature_importance
|
||||
|
||||
model_training >> model_pickle
|
||||
feature_encoding >> preprocessing_info
|
||||
feature_alignment >> preprocessing_info
|
||||
|
||||
performance_metrics >> accuracy_report
|
||||
confusion_matrix >> class_performance
|
||||
@@ -1,71 +0,0 @@
|
||||
"""
|
||||
Application Architecture Diagram
|
||||
Shows the modular Streamlit application structure and components
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.onprem.client import Users
|
||||
from diagrams.onprem.inmemory import Redis
|
||||
from diagrams.programming.framework import Django, React
|
||||
from diagrams.programming.language import Python
|
||||
from diagrams.generic.storage import Storage
|
||||
from diagrams.generic.compute import Rack
|
||||
from diagrams.onprem.network import Nginx
|
||||
|
||||
with Diagram("Streamlit Application Architecture",
|
||||
filename="diagrams/04_application_architecture",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# Users
|
||||
users = Users("End Users\n• Students\n• Educators\n• Administrators")
|
||||
|
||||
# Web Interface
|
||||
streamlit_frontend = Django("Streamlit Frontend\n• Rose Pine Theme\n• Responsive UI\n• Interactive Forms")
|
||||
|
||||
# Authentication Layer
|
||||
with Cluster("Authentication System"):
|
||||
auth_service = Redis("Authentication Service\n• Session management\n• Role-based access\n• Security validation")
|
||||
user_roles = Rack("User Roles\n• Admin (full access)\n• Educator (analytics)\n• User (predictions)")
|
||||
|
||||
# Application Core
|
||||
with Cluster("Application Core (app.py)"):
|
||||
main_app = Python("Main Application\n• Request routing\n• Session handling\n• Error management")
|
||||
auth_decorator = Python("Auth Decorator\n• @require_auth\n• Permission checks\n• Session validation")
|
||||
|
||||
# Configuration Layer
|
||||
with Cluster("Configuration (src/config/)"):
|
||||
theme_config = React("Theme Configuration\n• Rose Pine CSS\n• Color constants\n• UI styling")
|
||||
mappings_config = Storage("Feature Mappings\n• Human-readable labels\n• Preset templates\n• Validation rules")
|
||||
|
||||
# Utility Services
|
||||
with Cluster("Utility Services (src/utils/)"):
|
||||
model_utils = Python("Model Utilities\n• Model loading\n• Preprocessing\n• Prediction logic")
|
||||
auth_utils = Python("Auth Utilities\n• User validation\n• Session management\n• Password hashing")
|
||||
|
||||
# UI Components
|
||||
with Cluster("UI Components (src/components/)"):
|
||||
form_components = React("Form Components\n• Input sections\n• Validation feedback\n• Preset selection")
|
||||
prediction_display = React("Prediction Display\n• Results visualization\n• Confidence scores\n• Recommendations")
|
||||
|
||||
# Model Layer
|
||||
with Cluster("Model Layer"):
|
||||
trained_model = Storage("Trained Model\nrandom_forest_model.pkl\n93.84% accuracy")
|
||||
preprocessing_pipeline = Storage("Preprocessing Pipeline\npreprocessing_info.json\nFeature transformations")
|
||||
|
||||
# Data Flow
|
||||
users >> streamlit_frontend
|
||||
streamlit_frontend >> auth_service >> user_roles
|
||||
auth_service >> main_app >> auth_decorator
|
||||
|
||||
main_app >> theme_config
|
||||
main_app >> mappings_config
|
||||
main_app >> form_components
|
||||
main_app >> prediction_display
|
||||
|
||||
form_components >> model_utils
|
||||
model_utils >> trained_model
|
||||
model_utils >> preprocessing_pipeline
|
||||
model_utils >> prediction_display
|
||||
|
||||
auth_decorator >> auth_utils
|
||||
@@ -1,87 +0,0 @@
|
||||
"""
|
||||
Deployment Architecture Diagram
|
||||
Shows the deployment options and infrastructure for the application
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.onprem.client import Users
|
||||
from diagrams.saas.cdn import Cloudflare
|
||||
from diagrams.programming.framework import Django
|
||||
from diagrams.onprem.container import Docker
|
||||
from diagrams.k8s.compute import Pod
|
||||
from diagrams.generic.storage import Storage
|
||||
from diagrams.generic.network import Internet
|
||||
from diagrams.aws.compute import EC2
|
||||
from diagrams.gcp.compute import GKE
|
||||
|
||||
with Diagram("Deployment Architecture Options",
|
||||
filename="diagrams/05_deployment_architecture",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# Users
|
||||
users = Users("Global Users")
|
||||
internet = Internet("Internet")
|
||||
|
||||
# Deployment Option 1: Streamlit Cloud
|
||||
with Cluster("Option 1: Streamlit Cloud (Recommended)"):
|
||||
streamlit_cloud = Django("Streamlit Cloud\n• Automatic deployment\n• GitHub integration\n• SSL/HTTPS\n• Global CDN")
|
||||
github_repo = Storage("GitHub Repository\n• Source code\n• Model artifacts\n• Configuration")
|
||||
|
||||
# Cloud features
|
||||
cloud_features = [
|
||||
Storage("Automatic Scaling"),
|
||||
Storage("Monitoring & Logs"),
|
||||
Storage("Custom Domain Support")
|
||||
]
|
||||
|
||||
# Deployment Option 2: Docker Container
|
||||
with Cluster("Option 2: Docker Deployment"):
|
||||
docker_container = Docker("Docker Container\n• Streamlit app\n• Python runtime\n• Dependencies")
|
||||
container_registry = Storage("Container Registry\n• Docker Hub\n• AWS ECR\n• GCR")
|
||||
|
||||
# Container orchestration options
|
||||
with Cluster("Orchestration Options"):
|
||||
kubernetes = Pod("Kubernetes\n• Pod management\n• Auto-scaling\n• Load balancing")
|
||||
cloud_run = GKE("Cloud Run/Fargate\n• Serverless containers\n• Pay-per-use\n• Auto-scaling")
|
||||
|
||||
# Deployment Option 3: Traditional Server
|
||||
with Cluster("Option 3: Traditional Server"):
|
||||
web_server = EC2("Web Server\n• EC2/Compute Engine\n• Nginx reverse proxy\n• SSL certificate")
|
||||
app_server = Django("Application Server\n• Streamlit process\n• PM2/Supervisor\n• Process management")
|
||||
|
||||
# Shared Resources
|
||||
with Cluster("Shared Resources"):
|
||||
model_storage = Storage("Model Storage\n• Trained models\n• Preprocessing info\n• Version control")
|
||||
monitoring = Storage("Monitoring\n• Application logs\n• Performance metrics\n• Error tracking")
|
||||
backup = Storage("Backup & Recovery\n• Model versioning\n• Configuration backup\n• Disaster recovery")
|
||||
|
||||
# Data Flow
|
||||
users >> internet
|
||||
|
||||
# Streamlit Cloud flow
|
||||
internet >> streamlit_cloud
|
||||
github_repo >> streamlit_cloud
|
||||
streamlit_cloud >> cloud_features[0]
|
||||
streamlit_cloud >> cloud_features[1]
|
||||
streamlit_cloud >> cloud_features[2]
|
||||
|
||||
# Docker flow
|
||||
internet >> docker_container
|
||||
container_registry >> docker_container
|
||||
docker_container >> kubernetes
|
||||
docker_container >> cloud_run
|
||||
|
||||
# Traditional server flow
|
||||
internet >> web_server >> app_server
|
||||
|
||||
# Shared resources connections
|
||||
streamlit_cloud >> model_storage
|
||||
docker_container >> model_storage
|
||||
app_server >> model_storage
|
||||
|
||||
streamlit_cloud >> monitoring
|
||||
kubernetes >> monitoring
|
||||
app_server >> monitoring
|
||||
|
||||
model_storage >> backup
|
||||
@@ -1,70 +0,0 @@
|
||||
"""
|
||||
Feature Engineering Pipeline Diagram
|
||||
Shows the detailed feature transformation and encoding process
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.generic.storage import Storage
|
||||
from diagrams.generic.compute import Rack
|
||||
from diagrams.programming.language import Python
|
||||
from diagrams.onprem.analytics import Tableau
|
||||
from diagrams.generic.database import SQL
|
||||
|
||||
with Diagram("Feature Engineering Pipeline",
|
||||
filename="diagrams/06_feature_engineering_pipeline",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# Raw Features Input
|
||||
raw_features = Storage("Raw Features (37)\n• 36 predictors\n• 1 target variable\n• Mixed data types")
|
||||
|
||||
# Categorical Features Processing
|
||||
with Cluster("Categorical Feature Processing"):
|
||||
categorical_features = Rack("Categorical Features\n• Marital status\n• Application mode\n• Course\n• Nationality\n• Mother's/Father's qualification\n• Mother's/Father's occupation\n• Displaced\n• Educational special needs\n• Debtor\n• Tuition fees up to date\n• Gender\n• Scholarship holder\n• International")
|
||||
|
||||
label_encoding = Python("Label Encoding\n• Target variable\n• Dropout = 0\n• Enrolled = 1\n• Graduate = 2")
|
||||
|
||||
one_hot_encoding = Python("One-Hot Encoding\n• Categorical predictors\n• Binary columns creation\n• Feature expansion")
|
||||
|
||||
# Numerical Features Processing
|
||||
with Cluster("Numerical Feature Processing"):
|
||||
numerical_features = Rack("Numerical Features\n• Application order\n• Previous qualification grade\n• Admission grade\n• Age at enrollment\n• Curricular units (1st & 2nd sem)\n• Unemployment rate\n• Inflation rate\n• GDP")
|
||||
|
||||
feature_validation = Python("Feature Validation\n• Range checks\n• Outlier detection\n• Data type validation")
|
||||
|
||||
feature_scaling = Python("Feature Scaling\n• Optional for tree models\n• Standardization available\n• Normalization available")
|
||||
|
||||
# Feature Engineering Output
|
||||
with Cluster("Engineered Features"):
|
||||
encoded_features = Storage("Encoded Features (~105)\n• Original numerical (14)\n• One-hot encoded categorical (90+)\n• Label encoded target (1)")
|
||||
|
||||
feature_mapping = SQL("Feature Mapping\n• Original → Encoded names\n• Category mappings\n• Reverse mappings")
|
||||
|
||||
preprocessing_metadata = SQL("Preprocessing Metadata\n• Encoding schemes\n• Feature names\n• Transformation info")
|
||||
|
||||
# Quality Assurance
|
||||
with Cluster("Quality Assurance"):
|
||||
feature_consistency = Rack("Feature Consistency\n• Column order preservation\n• Missing feature handling\n• Shape validation")
|
||||
|
||||
data_validation = Rack("Data Validation\n• No missing values\n• Consistent data types\n• Feature alignment")
|
||||
|
||||
pipeline_testing = Tableau("Pipeline Testing\n• Train/test consistency\n• Transformation verification\n• Edge case handling")
|
||||
|
||||
# Data Flow
|
||||
raw_features >> categorical_features >> one_hot_encoding
|
||||
raw_features >> numerical_features >> feature_validation
|
||||
|
||||
categorical_features >> label_encoding
|
||||
numerical_features >> feature_scaling
|
||||
|
||||
one_hot_encoding >> encoded_features
|
||||
label_encoding >> encoded_features
|
||||
feature_validation >> encoded_features
|
||||
feature_scaling >> encoded_features
|
||||
|
||||
encoded_features >> feature_mapping
|
||||
encoded_features >> preprocessing_metadata
|
||||
|
||||
encoded_features >> feature_consistency
|
||||
feature_consistency >> data_validation
|
||||
data_validation >> pipeline_testing
|
||||
@@ -1,79 +0,0 @@
|
||||
"""
|
||||
User Journey and Interaction Flow Diagram
|
||||
Shows how different user types interact with the application
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.onprem.client import Users, User
|
||||
from diagrams.programming.framework import Django
|
||||
from diagrams.onprem.inmemory import Redis
|
||||
from diagrams.generic.compute import Rack
|
||||
from diagrams.onprem.analytics import Tableau
|
||||
from diagrams.generic.storage import Storage
|
||||
|
||||
with Diagram("User Journey and Interaction Flow",
|
||||
filename="diagrams/07_user_journey_flow",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# User Types
|
||||
with Cluster("User Types"):
|
||||
student_user = User("Student\n• Personal predictions\n• Academic planning\n• Risk assessment")
|
||||
educator_user = User("Educator\n• Student analytics\n• Intervention planning\n• Batch predictions")
|
||||
admin_user = User("Administrator\n• System management\n• User oversight\n• Full access")
|
||||
|
||||
# Authentication Flow
|
||||
with Cluster("Authentication Flow"):
|
||||
login_page = Django("Login Page\n• Credential input\n• Role selection\n• Security validation")
|
||||
auth_service = Redis("Authentication Service\n• Session creation\n• Role assignment\n• Access control")
|
||||
session_management = Redis("Session Management\n• Timeout handling\n• Security checks\n• State persistence")
|
||||
|
||||
# Application Interface
|
||||
with Cluster("Application Interface"):
|
||||
main_dashboard = Django("Main Dashboard\n• Role-based UI\n• Feature access\n• Navigation menu")
|
||||
|
||||
# Student workflow
|
||||
with Cluster("Student Workflow"):
|
||||
personal_form = Rack("Personal Information Form\n• Basic demographics\n• Academic history\n• Family background")
|
||||
academic_form = Rack("Academic Performance Form\n• Current grades\n• Curricular units\n• Progress tracking")
|
||||
prediction_results = Tableau("Prediction Results\n• Outcome probability\n• Confidence scores\n• Recommendations")
|
||||
|
||||
# Educator workflow
|
||||
with Cluster("Educator Workflow"):
|
||||
student_management = Rack("Student Management\n• Batch input\n• Template selection\n• Data validation")
|
||||
analytics_dashboard = Tableau("Analytics Dashboard\n• Class performance\n• Risk indicators\n• Intervention alerts")
|
||||
reporting_tools = Tableau("Reporting Tools\n• Performance reports\n• Trend analysis\n• Export capabilities")
|
||||
|
||||
# Admin workflow
|
||||
with Cluster("Admin Workflow"):
|
||||
user_management = Rack("User Management\n• Account creation\n• Role assignment\n• Access control")
|
||||
system_monitoring = Tableau("System Monitoring\n• Usage statistics\n• Performance metrics\n• Error logs")
|
||||
model_management = Storage("Model Management\n• Version control\n• Performance tracking\n• Updates")
|
||||
|
||||
# Prediction Engine
|
||||
with Cluster("Prediction Engine"):
|
||||
input_validation = Rack("Input Validation\n• Data quality checks\n• Range validation\n• Required fields")
|
||||
feature_processing = Rack("Feature Processing\n• Encoding\n• Transformation\n• Alignment")
|
||||
model_inference = Storage("Model Inference\n• Random Forest prediction\n• Confidence calculation\n• Result formatting")
|
||||
|
||||
# User Journey Flows
|
||||
|
||||
# Student journey
|
||||
student_user >> login_page >> auth_service >> main_dashboard
|
||||
main_dashboard >> personal_form >> academic_form
|
||||
academic_form >> input_validation >> feature_processing >> model_inference
|
||||
model_inference >> prediction_results
|
||||
|
||||
# Educator journey
|
||||
educator_user >> login_page >> auth_service >> main_dashboard
|
||||
main_dashboard >> student_management >> analytics_dashboard
|
||||
student_management >> input_validation >> feature_processing >> model_inference
|
||||
model_inference >> reporting_tools
|
||||
|
||||
# Admin journey
|
||||
admin_user >> login_page >> auth_service >> main_dashboard
|
||||
main_dashboard >> user_management >> system_monitoring >> model_management
|
||||
|
||||
# Session management connections
|
||||
auth_service >> session_management
|
||||
session_management >> main_dashboard
|
||||
@@ -1,93 +0,0 @@
|
||||
"""
|
||||
Security and Authentication Architecture Diagram
|
||||
Shows the comprehensive security model and authentication flow
|
||||
"""
|
||||
|
||||
from diagrams import Diagram, Cluster, Edge
|
||||
from diagrams.onprem.client import Users
|
||||
from diagrams.onprem.security import Oauth
|
||||
from diagrams.onprem.inmemory import Redis
|
||||
from diagrams.programming.language import Python
|
||||
from diagrams.generic.compute import Rack
|
||||
from diagrams.generic.storage import Storage
|
||||
from diagrams.onprem.network import Nginx
|
||||
from diagrams.generic.database import SQL
|
||||
|
||||
with Diagram("Security and Authentication Architecture",
|
||||
filename="diagrams/08_security_architecture",
|
||||
show=False,
|
||||
direction="TB"):
|
||||
|
||||
# External Users
|
||||
users = Users("External Users\n• Students\n• Educators\n• Administrators")
|
||||
|
||||
# Security Perimeter
|
||||
with Cluster("Security Perimeter"):
|
||||
https_gateway = Nginx("HTTPS Gateway\n• SSL/TLS encryption\n• Certificate management\n• Secure headers")
|
||||
rate_limiting = Rack("Rate Limiting\n• Request throttling\n• DDoS protection\n• Abuse prevention")
|
||||
|
||||
# Authentication Layer
|
||||
with Cluster("Authentication System"):
|
||||
login_handler = Oauth("Login Handler\n• Credential validation\n• Multi-factor support\n• Brute force protection")
|
||||
|
||||
password_security = Python("Password Security\n• Hashing (bcrypt/scrypt)\n• Salt generation\n• Strength validation")
|
||||
|
||||
session_store = Redis("Session Store\n• Secure session tokens\n• Expiration management\n• Cross-site protection")
|
||||
|
||||
# Authorization Layer
|
||||
with Cluster("Authorization System"):
|
||||
role_manager = Python("Role Manager\n• Role definitions\n• Permission mapping\n• Access control")
|
||||
|
||||
permission_engine = Rack("Permission Engine\n• Route protection\n• Resource access\n• Action authorization")
|
||||
|
||||
with Cluster("User Roles & Permissions"):
|
||||
admin_permissions = Storage("Admin Permissions\n• Full system access\n• User management\n• Model updates\n• System configuration")
|
||||
|
||||
educator_permissions = Storage("Educator Permissions\n• Student analytics\n• Batch predictions\n• Reporting tools\n• Limited admin")
|
||||
|
||||
user_permissions = Storage("User Permissions\n• Personal predictions\n• Profile management\n• Basic analytics\n• Read-only access")
|
||||
|
||||
# Security Middleware
|
||||
with Cluster("Security Middleware"):
|
||||
auth_decorator = Python("Auth Decorator\n• @require_auth\n• Route protection\n• Session validation")
|
||||
|
||||
input_sanitization = Rack("Input Sanitization\n• XSS prevention\n• SQL injection protection\n• Data validation")
|
||||
|
||||
csrf_protection = Rack("CSRF Protection\n• Token validation\n• State verification\n• Request authentication")
|
||||
|
||||
# Application Security
|
||||
with Cluster("Application Security"):
|
||||
secure_storage = Storage("Secure Storage\n• Encrypted data\n• Access logs\n• Audit trails")
|
||||
|
||||
error_handling = Rack("Error Handling\n• Secure error messages\n• Log sanitization\n• Information disclosure prevention")
|
||||
|
||||
monitoring = SQL("Security Monitoring\n• Failed login attempts\n• Suspicious activity\n• Access pattern analysis")
|
||||
|
||||
# Data Protection
|
||||
with Cluster("Data Protection"):
|
||||
data_encryption = Storage("Data Encryption\n• At-rest encryption\n• In-transit encryption\n• Key management")
|
||||
|
||||
privacy_controls = Rack("Privacy Controls\n• Data anonymization\n• PII protection\n• Consent management")
|
||||
|
||||
backup_security = Storage("Backup Security\n• Encrypted backups\n• Access controls\n• Recovery procedures")
|
||||
|
||||
# Security Flow
|
||||
users >> https_gateway >> rate_limiting >> login_handler
|
||||
login_handler >> password_security >> session_store
|
||||
|
||||
session_store >> role_manager >> permission_engine
|
||||
|
||||
role_manager >> admin_permissions
|
||||
role_manager >> educator_permissions
|
||||
role_manager >> user_permissions
|
||||
|
||||
permission_engine >> auth_decorator >> input_sanitization
|
||||
input_sanitization >> csrf_protection
|
||||
|
||||
auth_decorator >> secure_storage
|
||||
csrf_protection >> error_handling
|
||||
error_handling >> monitoring
|
||||
|
||||
secure_storage >> data_encryption
|
||||
data_encryption >> privacy_controls
|
||||
privacy_controls >> backup_security
|
||||
@@ -1,82 +0,0 @@
|
||||
"""
|
||||
Master Script to Generate All Architecture Diagrams
|
||||
Run this script to generate all diagram images at once
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
def generate_diagram(script_name):
|
||||
"""Generate a single diagram by running its Python script"""
|
||||
try:
|
||||
print(f"Generating {script_name}...")
|
||||
result = subprocess.run([sys.executable, script_name],
|
||||
capture_output=True, text=True, cwd="diagrams")
|
||||
|
||||
if result.returncode == 0:
|
||||
print(f"✅ Successfully generated {script_name}")
|
||||
else:
|
||||
print(f"❌ Error generating {script_name}:")
|
||||
print(result.stderr)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Exception while generating {script_name}: {e}")
|
||||
|
||||
def main():
|
||||
"""Generate all architecture diagrams"""
|
||||
|
||||
# Check if diagrams library is installed
|
||||
try:
|
||||
import diagrams
|
||||
print("✅ Diagrams library found")
|
||||
except ImportError:
|
||||
print("❌ Diagrams library not found. Installing...")
|
||||
subprocess.run([sys.executable, "-m", "pip", "install", "-r", "diagrams/requirements.txt"])
|
||||
print("✅ Diagrams library installed")
|
||||
|
||||
# List of diagram scripts to generate
|
||||
diagram_scripts = [
|
||||
"00_system_overview.py",
|
||||
"01_data_flow_architecture.py",
|
||||
"02_eda_process_flow.py",
|
||||
"03_ml_pipeline_architecture.py",
|
||||
"04_application_architecture.py",
|
||||
"05_deployment_architecture.py",
|
||||
"06_feature_engineering_pipeline.py",
|
||||
"07_user_journey_flow.py",
|
||||
"08_security_architecture.py"
|
||||
]
|
||||
|
||||
print("🚀 Starting diagram generation...")
|
||||
print("=" * 50)
|
||||
|
||||
# Create diagrams directory if it doesn't exist
|
||||
Path("diagrams").mkdir(exist_ok=True)
|
||||
|
||||
# Generate each diagram
|
||||
for script in diagram_scripts:
|
||||
if os.path.exists(f"diagrams/{script}"):
|
||||
generate_diagram(script)
|
||||
else:
|
||||
print(f"⚠️ Script not found: {script}")
|
||||
|
||||
print("=" * 50)
|
||||
print("🎉 Diagram generation complete!")
|
||||
print("\nGenerated diagrams:")
|
||||
|
||||
# List generated PNG files
|
||||
diagrams_dir = Path("diagrams")
|
||||
png_files = list(diagrams_dir.glob("*.png"))
|
||||
|
||||
if png_files:
|
||||
for png_file in sorted(png_files):
|
||||
print(f" 📊 {png_file.name}")
|
||||
else:
|
||||
print(" ⚠️ No PNG files found. Check for errors above.")
|
||||
|
||||
print(f"\nTotal diagrams generated: {len(png_files)}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user