Based on the logs and code, I can see the issue is with the socket connection and progress tracking. The process shows as completed but gets stuck because the socket connection isn't properly maintained. Here's the fix: from flask import Flask, render_template, request, redirect, url_for, flash from flask_socketio import SocketIO, emit import os from threading import Thread, Lock from werkzeug.utils import secure_filename from utils.db_operations import main_process_excel_to_sql, main_process_sql_to_sql, get_db_connection from utils.model_training import train_all_models, train_prediction_model, train_class_models from utils.data_loader import load_and_prepare_data from utils.config import * from utils.progress_tracker import ProgressTracker, set_global_socketio app = Flask(__name__) app.secret_key = 'your_secret_key_here' app.config['UPLOAD_FOLDER'] = 'uploads' app.config['ALLOWED_EXTENSIONS'] = {'xlsx', 'xls'} os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True) # Initialize SocketIO with async_mode and cors_allowed_origins socketio = SocketIO(app, async_mode='threading', cors_allowed_origins="*") set_global_socketio(socketio) # Thread lock for socket operations socket_lock = Lock() # -------------------------------------------------------------------------- # Helpers # -------------------------------------------------------------------------- def allowed_file(filename): return '.' in filename and filename.rsplit('.', 1)[1].lower() in app.config['ALLOWED_EXTENSIONS'] # -------------------------------------------------------------------------- # Excel → SQL ETL # -------------------------------------------------------------------------- def run_excel_import(filepath): namespace = '/insert_excel' try: with socket_lock: socketio.emit('progress', {'step': 0, 'message': 'Starting Excel import...'}, namespace=namespace) main_process_excel_to_sql( excel_path=filepath, odbc_driver=TGT_ODBC_DRIVER, sql_server_instance=TGT_SERVER, database_name=TGT_DATABASE, trusted_connection=TGT_USE_TRUSTED, sql_username=TGT_USERNAME, sql_password=TGT_PASSWORD ) with socket_lock: socketio.emit('complete', {'message': 'Excel import completed successfully'}, namespace=namespace) except Exception as e: with socket_lock: socketio.emit('error', {'message': str(e)}, namespace=namespace) @socketio.on('start_excel', namespace='/insert_excel') def on_start_excel(data): Thread(target=run_excel_import, args=(data['filepath'],)).start() # -------------------------------------------------------------------------- # SQL → SQL ETL # -------------------------------------------------------------------------- def run_db_import(details): namespace = '/insert_db' try: with socket_lock: socketio.emit('progress', {'step': 0, 'message': 'Starting DB import...'}, namespace=namespace) main_process_sql_to_sql(**details) with socket_lock: socketio.emit('complete', {'message': 'DB import completed successfully'}, namespace=namespace) except Exception as e: with socket_lock: socketio.emit('error', {'message': str(e)}, namespace=namespace) @socketio.on('start_db', namespace='/insert_db') def on_start_db(data): Thread(target=run_db_import, args=(data,)).start() # -------------------------------------------------------------------------- # Model Training ETL # -------------------------------------------------------------------------- _preprocessed_df = None def run_training(mode): global _preprocessed_df namespace = '/model_training' try: with socket_lock: socketio.emit('progress', {'step': 0, 'message': 'Starting training...'}, namespace=namespace) if mode in ('preprocess', 'all'): _preprocessed_df = load_and_prepare_data() if mode in ('prediction', 'all'): if _preprocessed_df is None: _preprocessed_df = load_and_prepare_data() train_prediction_model(_preprocessed_df) if mode in ('classification', 'all'): if _preprocessed_df is None: _preprocessed_df = load_and_prepare_data() train_class_models(_preprocessed_df) with socket_lock: socketio.emit('complete', {'message': 'Training completed successfully'}, namespace=namespace) except Exception as e: with socket_lock: socketio.emit('error', {'message': str(e)}, namespace=namespace) @socketio.on('connect', namespace='/model_training') def on_connect(): app.logger.info("Client connected to /model_training; waiting for start signal.") @socketio.on('start_training', namespace='/model_training') def on_start_training(data): mode = data.get('mode', 'all') Thread(target=run_training, args=(mode,)).start() # -------------------------------------------------------------------------- # Flask Routes (remain unchanged) # -------------------------------------------------------------------------- @app.route('/') def home(): return render_template('index.html') @app.route('/insert-data') def insert_data(): return render_template('insert_data.html') @app.route('/view-reports', methods=['GET','POST']) def view_reports(): # 1) your lists all_pav = ['TO0-CO0', 'NA', 'TO0-CO1', 'TO1-CO0', 'TO0-CO2', 'TO1-CO1', 'TO1-CO2', 'TO2-CO0', 'TO2-CO1', 'TO2-CO2', 'TO0-CO3', 'TO3-CO0', 'TO3-CO1', 'TO3-CO2', 'TO3-CO3', 'TO1-CO3', 'TO0-PO1', 'TO0-PO0', 'TO1-PO1', 'TO1-PO0', 'TO0-PO2', 'NV-PO1', 'TO2-PO2', 'TO2-PO0', 'TO1-PO2', 'TO3-PO2', 'TO2-PO1', 'TO3-PO0', 'TO0-PO3', 'TO1-PO3', 'TO2-PO3', 'TO3-PO1', 'TO3-PO3', 'NV-CO0', 'NV-CO1', 'NV-CO2', 'NV-CO3', 'NV-NV', 'TO0-NV', 'TO1-NV', 'TO2-CO3', 'TO2-NV', 'TO3-NV','NA'] all_open = all_pav all_black = [" NV-NV", "BT0-BC0", "BT0-BC1", "BT0-BC2", "BT0-BC3", "BT0-NV", "BT1-BC0", "BT1-BC1", "BT1-BC2", "BT1-BC3", "BT1-NV", "BT2-BC0", "BT2-BC1", "BT2-BC2", "BT2-BC3", "BT2-NV", "BT3-BC0", "BT3-BC1", "BT3-BC2", "BT3-BC3", "BT3-NV", "NV-BC0", "NV-BC1", "NV-BC2", "NV-BC3",'NA'] all_white = ['WT0-NV','WT0-WC0','WT0-WC1','WT0-WC2','WT0-WC3','WT1-NV','WT1-WC0','WT1-WC1','WT1-WC2','WT1-WC3','WT2-NV','WT2-WC0', 'WT2-WC1','WT2-WC2','WT2-WC3','WT3-NV','WT3-WC0','WT3-WC1','WT3-WC2','WT3-WC3','NV-NV','NV-WC0','NV-WC1','NV-WC2','NV-WC3','NA'] all_shape = ['NA','AS','CB','CU','EM','HR','LR','MQ','OV','PE','PR','R','RD','RQ','TR'] all_cut =['NA','EX1','EX2','EX3','EX4','FG1','GD1','GD2','VG1','VG2'] all_quality = ['NA','FL','I1','I1_','I2','I2_','I3','I3_','I4','I4_','I5','I5_','I6','I7','I8','IF','SI1','SI2','SI2_','SI2M','SI3','VS1','VS2','VVS1','VVS2',] all_color = ['D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','NA'] # 2) map category → (table_name, column_name, value_list) CATEGORY_MAP = { 'Blk': ('Eng_Mkbl_change_Blk_tbl', 'original_Blk', all_black), 'Wht': ('Eng_Mkbl_change_Wht_tbl', 'original_Wht', all_white), 'Pav': ('Eng_Mkblchange_Pav_tbl', 'original_Pav', all_pav), 'Open': ('Eng_Mkbl_change_Open_tbl', 'original_Open', all_open), 'Shp': ('Eng_Mkbl_change_Shp_tbl', 'original_Shp', all_shape), 'Cut': ('Eng_Mkbl_change_Cut_tbl', 'original_Cut', all_cut), 'Col': ('Eng_Mkbl_change_Col_tbl', 'original_Col', all_color), 'Qua': ('Eng_Mkbl_change_Qua_tbl', 'original_Qua', all_quality), } # 3) pretty labels CATEGORY_LABELS = { 'Blk':'Black', 'Wht':'White', 'Pav':'Pav', 'Open':'Open', 'Shp':'Shape','Cut':'Cut','Col':'Color','Qua':'Quality' } #----------------------------------------------------------------------------- selected_category = request.form.get('category') # e.g. 'blk' selected_value = request.form.get('value') # e.g. 'BT2-BC0' results = None if request.method == 'POST' and selected_category and selected_value: table, column, _ = CATEGORY_MAP[selected_category] # build & run parameterized query conn = get_db_connection( TGT_ODBC_DRIVER, TGT_SERVER, TGT_DATABASE, TGT_USE_TRUSTED, TGT_USERNAME, TGT_PASSWORD ) sql = f""" SELECT *, ROUND([change %],2) AS [change %] FROM dbo.{table} WHERE {column} = ? ORDER BY [count] DESC """ cursor = conn.execute(sql, selected_value) cols = [c[0] for c in cursor.description] rows = cursor.fetchall() # convert each row to dict {col: value} results = [dict(zip(cols, r)) for r in rows] conn.close() return render_template( 'view_reports.html', category_labels=CATEGORY_LABELS, all_values={k: v for (k, (_,_,v)) in CATEGORY_MAP.items()}, selected_category=selected_category, selected_value=selected_value, results=results ) @app.route('/model-training') def model_training(): return render_template('model_training.html') @app.route('/model-prediction') def model_prediction(): return render_template('model_prediction.html') @app.route('/upload-excel', methods=['POST']) def upload_excel(): if 'file' not in request.files: flash('No file part', 'error') return redirect(url_for('insert_data')) file = request.files['file'] if file.filename == '': flash('No selected file', 'error') return redirect(url_for('insert_data')) if file and allowed_file(file.filename): filename = secure_filename(file.filename) filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename) file.save(filepath) socketio.emit('start_excel', {'filepath': filepath}, namespace='/insert_excel') flash('Upload queued for processing.', 'info') return redirect(url_for('insert_data')) flash('Invalid file type', 'error') return redirect(url_for('insert_data')) @app.route('/connect-database', methods=['POST']) def connect_database(): details = request.form.to_dict() socketio.emit('start_db', details, namespace='/insert_db') flash('Database import queued for processing.', 'info') return redirect(url_for('insert_data')) # -------------------------------------------------------------------------- # Run # -------------------------------------------------------------------------- if __name__ == '__main__': socketio.run(app, debug=True, allow_unsafe_werkzeug=True) {% extends "base.html" %} {% block content %}

Data Import

Import from Excel

Import from Database

ODBC Driver

Server

Database

Table Name

Use Trusted Connection

{% endblock %} {% block scripts %} {% endblock %} Key changes made: 1. Added explicit SocketIO configuration with async_mode and CORS 2. Added thread locking for socket operations to prevent race conditions 3. Improved progress tracking with explicit start/complete/error events 4. Added proper socket.io client-side implementation in the template 5. Added visual progress bar and status messages 6. Ensured proper cleanup of progress indicators 7. Added allow_unsafe_werkzeug=True for better compatibility These changes should resolve the socket connection issues and provide better progress tracking.