I'll help you reorganize the code into a more modular structure with specific functions to avoid reloading data in Flask. Here's the improved version: # Database Configuration ODBC_DRIVER = "ODBC Driver 17 for SQL Server" SQL_SERVER_INSTANCE = r"DESKTOP-EQL9MBT\SQLEXPRESS" DATABASE_NAME = "MyData Warehouse" TRUSTED_CONNECTION = True SQL_USERNAME = None SQL_PASSWORD = None TABLE_NAME = "processed_diamond_data" PROCESSED_TABLE_NAME = "processed_diamond_data" # Versioning VERSION_STRING = datetime.now().strftime("%Y%m%d_%H%M%S") import pandas as pd import pyodbc from config import * from db_operations import get_db_connection def load_data(): """Load data from SQL Server and return a DataFrame""" src_conn = get_db_connection( ODBC_DRIVER, SQL_SERVER_INSTANCE, DATABASE_NAME, TRUSTED_CONNECTION, SQL_USERNAME, SQL_PASSWORD ) query = f"SELECT * FROM {TABLE_NAME};" df = pd.read_sql(query, src_conn) src_conn.close() return df def preprocess_data(df): """Preprocess the raw data""" # Convert numeric columns numeric_cols = ['MkblAmt','EngAmt','GrdAmt','ByGrdAmt','GiaAmt', 'EngGraphCts','EngCts','MkblCts','GrdCts','ByGrdCts','GiaCts'] for col in numeric_cols: df[col] = pd.to_numeric(df[col], errors='coerce') # Calculate percentage changes df['pct_change_mkble'] = ((df['MkblAmt'] - df['EngAmt']) / df['EngAmt'] * 100).round(0) df['pct_change_grd'] = ((df['GrdAmt'] - df['EngAmt']) / df['EngAmt'] * 100).round(0) df['pct_change_bygrd'] = ((df['ByGrdAmt'] - df['EngAmt']) / df['EngAmt'] * 100).round(0) df['pct_change_gia'] = ((df['GiaAmt'] - df['EngAmt']) / df['EngAmt'] * 100).round(0) # Categorize GIA amount changes df['change_in_pct_gia_2'] = df.apply( lambda row: 'Positive amt change' if row['pct_change_gia'] > 3 else 'Negative amt change' if row['pct_change_gia'] < -3 else 'No significant change', axis=1 ) # Create percentage intervals edges = list(range(-50, 51, 5)) bins = [-np.inf] + edges + [np.inf] labels = ( ["Below –50%"] + [f"({a}%, {b}%]" for a, b in zip(edges[:-1], edges[1:])] + ["Above +50%"] ) df['pct_interval_gia'] = pd.cut( df['pct_change_gia'], bins=bins, labels=labels, right=True ) df['pct_interval_gia'] = df['pct_interval_gia'].cat.add_categories(["0%"]) df.loc[df['pct_change_gia'] == 0, 'pct_interval_gia'] = "0%" # Select relevant columns df = df[[ 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly', 'EngBlk', 'EngWht', 'EngOpen', 'EngPav', 'EngAmt', 'GiaCts', 'GiaShp', 'GiaQua', 'GiaCol', 'GiaCut', 'GiaPol', 'GiaSym', 'GiaFlo', 'GiaNts', 'GiaMikly', 'GiaBlk', 'GiaWht', 'GiaOpen', 'GiaPav', 'GiaAmt', 'Change_shape_Eng_to_Mkbl_value', 'Change_shape_Eng_to_Gia_value', 'Change_quality_Eng_to_Mkbl_value', 'Change_quality_Eng_to_Gia_value', 'Change_color_Eng_to_Mkbl_value', 'Change_color_Eng_to_Gia_value', 'Change_cut_Eng_to_Mkbl_value', 'Change_cut_Eng_to_Gia_value', 'Change_Blk_Eng_to_Mkbl_value', 'Change_Blk_Eng_to_Gia_value', 'Change_Wht_Eng_to_Mkbl_value', 'Change_Wht_Eng_to_Gia_value', 'Change_Open_Eng_to_Mkbl_value', 'Change_Open_Eng_to_Gia_value', 'Change_Pav_Eng_to_Mkbl_value', 'Change_Pav_Eng_to_Gia_value', 'pct_change_mkble', 'pct_change_grd', 'pct_change_bygrd', 'pct_change_gia', 'change_in_pct_gia_2', 'pct_interval_gia' ]] return df def apply_label_encoding(df): """Apply label encoding to categorical columns""" s = (df.dtypes == "object") object_cols = list(s[s].index) os.makedirs("Label_encoders", exist_ok=True) for col in object_cols: le = LabelEncoder() df[col] = le.fit_transform(df[col]) dump(le, f"Label_encoders/label_encoder_{col}.joblib") return df def clean_data(df): """Clean the data by removing zeros and duplicates""" df = df[(df[['GiaAmt', 'EngCts']] != 0).all(axis=1)] df[['EngAmt', 'GiaAmt']] = df[['EngAmt','GiaAmt']].round().astype('Int64') return df.drop_duplicates() import joblib import os import numpy as np import pandas as pd from sklearn.preprocessing import PowerTransformer, StandardScaler, OneHotEncoder from sklearn.compose import ColumnTransformer from sklearn.pipeline import Pipeline from sklearn.impute import SimpleImputer from sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier from imblearn.pipeline import Pipeline as ImbPipeline from config import VERSION_STRING def train_prediction_model(df): """Train the prediction model for GiaAmt""" ref_Y = df[df["GiaAmt"].notna()] ref_Y = ref_Y[[ 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngBlk', 'EngWht', 'EngOpen', 'EngPav', 'EngAmt', 'GiaAmt' ]] X = ref_Y.drop(columns=['GiaAmt']) y = ref_Y['GiaAmt'] y_log = np.log1p(y) # Handle skewness numeric_cols = X.select_dtypes(include=[np.number]).columns skew_values = X[numeric_cols].skew() high_skew_cols = skew_values[abs(skew_values) > 2.0].index.tolist() # Preprocessing pipeline power_transformer = Pipeline([ ('yeo_johnson', PowerTransformer(method='yeo-johnson')) ]) preprocessor = ColumnTransformer(transformers=[ ('power', power_transformer, high_skew_cols), ], remainder='passthrough') # Model pipelines pipeline_dt = Pipeline([ ('preprocessor', preprocessor), ('dt', DecisionTreeRegressor()) ]) pipeline_rf = Pipeline([ ('preprocessor', preprocessor), ('rf', RandomForestRegressor(n_estimators=150, random_state=42)) ]) # Parameter grids param_grid_dt = { 'dt__max_depth': [None, 5, 10, 20], 'dt__min_samples_split': [2, 5, 10], 'dt__min_samples_leaf': [1, 2, 4] } param_grid_rf = { 'rf__n_estimators': [100, 150, 200], 'rf__max_depth': [None, 10, 20], 'rf__min_samples_split': [2, 5, 10], 'rf__min_samples_leaf': [1, 2, 4], 'rf__max_features': ['sqrt', 'log2'] } # Grid search grids = { 'DecisionTree': (pipeline_dt, param_grid_dt), 'RandomForest': (pipeline_rf, param_grid_rf) } for name, (pipe, params) in grids.items(): grid = GridSearchCV(pipe, param_grid=params, scoring='neg_mean_absolute_error', cv=5, n_jobs=1) grid.fit(X, y_log) joblib.dump(grid.best_estimator_, f"Models/{name}_best_{VERSION_STRING}.pkl") def train_classification_model(df, target_column): """Train a classification model for a specific target column""" ref_Y = df.dropna() ref_Y = ref_Y[[ 'EngCts', 'EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngBlk', 'EngWht', 'EngOpen', 'EngPav', 'EngAmt', 'Change_shape_Eng_to_Mkbl_value', 'Change_shape_Eng_to_Gia_value', 'Change_quality_Eng_to_Mkbl_value', 'Change_quality_Eng_to_Gia_value', 'Change_color_Eng_to_Mkbl_value', 'Change_color_Eng_to_Gia_value', 'Change_cut_Eng_to_Mkbl_value', 'Change_cut_Eng_to_Gia_value', 'Change_Blk_Eng_to_Mkbl_value', 'Change_Blk_Eng_to_Gia_value', 'Change_Wht_Eng_to_Mkbl_value', 'Change_Wht_Eng_to_Gia_value', 'Change_Open_Eng_to_Mkbl_value', 'Change_Open_Eng_to_Gia_value', 'Change_Pav_Eng_to_Mkbl_value', 'Change_Pav_Eng_to_Gia_value' ]] X = ref_Y.drop([ 'Change_shape_Eng_to_Mkbl_value', 'Change_shape_Eng_to_Gia_value', 'Change_quality_Eng_to_Mkbl_value', 'Change_quality_Eng_to_Gia_value', 'Change_color_Eng_to_Mkbl_value', 'Change_color_Eng_to_Gia_value', 'Change_cut_Eng_to_Mkbl_value', 'Change_cut_Eng_to_Gia_value', 'Change_Blk_Eng_to_Mkbl_value', 'Change_Blk_Eng_to_Gia_value', 'Change_Wht_Eng_to_Mkbl_value', 'Change_Wht_Eng_to_Gia_value', 'Change_Open_Eng_to_Mkbl_value', 'Change_Open_Eng_to_Gia_value', 'Change_Pav_Eng_to_Mkbl_value', 'Change_Pav_Eng_to_Gia_value' ], axis=1) y = ref_Y[target_column] # Preprocessing categorical_features = ['EngShp', 'EngQua', 'EngCol', 'EngCut', 'EngPol', 'EngSym', 'EngFlo', 'EngNts', 'EngMikly'] numerical_features = ['EngCts', 'EngAmt'] t_num = Pipeline([ ('imputer', SimpleImputer(strategy='mean')), ('scaler', StandardScaler()) ]) t_cat = Pipeline([ ('imputer', SimpleImputer(strategy='most_frequent')), ('onehot', OneHotEncoder(handle_unknown='ignore')) ]) preprocessor = ColumnTransformer([ ('num', t_num, numerical_features), ('cat', t_cat, categorical_features) ]) # Models classifiers = { 'DT': DecisionTreeClassifier(class_weight='balanced', random_state=42), 'RF': RandomForestClassifier(n_estimators=150, class_weight='balanced', random_state=42) } pipelines = { name: ImbPipeline([ ('pre', preprocessor), ('clf', clf) ]) for name, clf in classifiers.items() } # Parameter grids grid_params = { 'DT': { 'clf__max_depth': [None, 10, 20], 'clf__min_samples_split': [2, 5] }, 'RF': { 'clf__n_estimators': [100, 150, 200], 'clf__max_features': ['sqrt', 'log2'] } } # Training cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42) os.makedirs('Models', exist_ok=True) for name, pipeline in pipelines.items(): gs = GridSearchCV( pipeline, param_grid=grid_params[name], scoring='f1_macro', cv=cv, n_jobs=2, verbose=1, error_score='raise' ) gs.fit(X, y) path = os.path.join('Models', f"{name}_{target_column}_{VERSION_STRING}.joblib") joblib.dump(gs.best_estimator_, path) def train_all_models(df): """Train all models (prediction and classification)""" os.makedirs("Models", exist_ok=True) # Train prediction model train_prediction_model(df) # Train classification models target_columns = [ 'Change_shape_Eng_to_Mkbl_value', 'Change_shape_Eng_to_Gia_value', 'Change_quality_Eng_to_Mkbl_value', 'Change_quality_Eng_to_Gia_value', 'Change_color_Eng_to_Mkbl_value', 'Change_color_Eng_to_Gia_value', 'Change_cut_Eng_to_Mkbl_value', 'Change_cut_Eng_to_Gia_value', 'Change_Blk_Eng_to_Mkbl_value', 'Change_Blk_Eng_to_Gia_value', 'Change_Wht_Eng_to_Mkbl_value', 'Change_Wht_Eng_to_Gia_value', 'Change_Open_Eng_to_Mkbl_value', 'Change_Open_Eng_to_Gia_value', 'Change_Pav_Eng_to_Mkbl_value', 'Change_Pav_Eng_to_Gia_value' ] for target in target_columns: train_classification_model(df, target) from data_loader import load_data, preprocess_data, apply_label_encoding, clean_data from model_training import train_all_models def initialize_models(): """Initialize all models by loading and preprocessing data once""" print("Loading and preprocessing data...") df = load_data() df = preprocess_data(df) df = apply_label_encoding(df) df = clean_data(df) print("Training models...") train_all_models(df) print("Model training completed!") return df if __name__ == "__main__": initialize_models() This reorganization: 1. Separates concerns into different modules: - config.py for configuration - data_loader.py for data loading and preprocessing - model_training.py for model training logic - main.py as the entry point 2. Creates a single initialization function that loads and preprocesses data once 3. Makes the code more modular and easier to maintain 4. Allows Flask to call initialize_models() once at startup rather than reloading data for each request 5. Follows Python best practices for code organization and structure The models will be trained once and saved to disk, and Flask can load them from there when needed.