PCM_Report/experiment_monitor.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
实验状态监控服务 - 在等待状态下定时查询InfluxDB数据，检测实验状态变化
"""
import threading
import time
import sqlite3
import datetime
from typing import Optional, Callable, Dict, Any
from logger import get_logger
from influx_service import InfluxService, InfluxConnectionParams
import pandas as pd

logger = get_logger()


class ExperimentStateMonitor:
    """实验状态监控器 - 定时查询InfluxDB数据并检测状态变化"""
    
    def __init__(
        self,
        experiment_id: int,
        work_order_no: str,
        start_time: datetime.datetime,
        influx_params: InfluxConnectionParams,
        query_config: Dict[str, Any],
        on_state_changed: Optional[Callable[[str, str], None]] = None,
        on_connection_changed: Optional[Callable[[bool, str], None]] = None,
        poll_interval: int = 5,
        max_retry_attempts: int = 3,
        retry_backoff_base: float = 2.0,
        max_consecutive_failures: int = 10
    ):
        """
        初始化监控器
        
        Args:
            experiment_id: 实验记录ID
            work_order_no: 工单号
            start_time: 工单填写时间（查询此时间之后的数据）
            influx_params: InfluxDB连接参数
            query_config: 查询配置，包含:
                - bucket: 数据桶名称
                - measurement: 测量名称
                - fields: 字段列表
                - filters: 过滤条件
                - status_field: 状态字段名称（用于检测状态变化）
                - status_values: 状态值配置 {'start': '开始值', 'end': '结束值'}
            on_state_changed: 状态变化回调函数 (old_state, new_state)
            on_connection_changed: 连接状态变化回调函数 (is_connected, message)
            poll_interval: 轮询间隔（秒）
            max_retry_attempts: 单次查询最大重试次数
            retry_backoff_base: 重试退避基数（指数退避）
            max_consecutive_failures: 最大连续失败次数（超过后报告连接断开）
        """
        self.experiment_id = experiment_id
        self.work_order_no = work_order_no
        self.start_time = start_time
        self.influx_params = influx_params
        self.query_config = query_config
        self.on_state_changed = on_state_changed
        self.on_connection_changed = on_connection_changed
        self.poll_interval = poll_interval
        self.max_retry_attempts = max_retry_attempts
        self.retry_backoff_base = retry_backoff_base
        self.max_consecutive_failures = max_consecutive_failures
        
        self._thread: Optional[threading.Thread] = None
        self._stop_event = threading.Event()
        self._last_state: Optional[str] = None
        self._experiment_started = False
        self._experiment_ended = False
        self._start_time_recorded: Optional[str] = None
        self._end_time_recorded: Optional[str] = None
        
        # 连接状态管理
        self._is_connected = False
        self._consecutive_failures = 0
        self._last_success_time: Optional[datetime.datetime] = None
        self._connection_lock = threading.Lock()
        
        logger.info(
            f"[监控器初始化] 实验ID={experiment_id}, 工单号={work_order_no}, "
            f"开始时间={start_time}, 轮询间隔={poll_interval}秒, "
            f"最大重试次数={max_retry_attempts}, 最大连续失败次数={max_consecutive_failures}"
        )
    
    def start(self) -> None:
        """启动监控线程"""
        if self._thread is not None and self._thread.is_alive():
            logger.warning(f"[监控器] 实验{self.experiment_id}的监控线程已在运行")
            return
        
        self._stop_event.clear()
        self._thread = threading.Thread(target=self._monitor_loop, daemon=True)
        self._thread.start()
        logger.info(f"[监控器] 实验{self.experiment_id}的监控线程已启动")
    
    def stop(self) -> None:
        """停止监控线程"""
        if self._thread is None:
            return
        
        self._stop_event.set()
        
        # 避免线程试图join自己的问题
        current_thread = threading.current_thread()
        if self._thread.is_alive() and self._thread != current_thread:
            self._thread.join(timeout=5)
        
        logger.info(f"[监控器] 实验{self.experiment_id}的监控线程已停止")
    
    def _monitor_loop(self) -> None:
        """监控循环 - 定时查询并检测状态变化"""
        try:
            logger.info(f"[监控循环] 开始监控实验{self.experiment_id}")
            
            while not self._stop_event.is_set():
                try:
                    # 查询当前状态
                    logger.debug(f"[监控循环] 实验{self.experiment_id}开始查询状态...")
                    current_state = self._query_current_state()
                    logger.info(
                        f"[监控循环] 实验{self.experiment_id}查询结果: 当前状态={current_state}, "
                        f"上一状态={self._last_state}"
                    )
                    
                    if current_state is not None:
                        
                        # 检测状态变化
                        if self._last_state is None:
                            # 第一次查询，记录初始状态
                            self._last_state = current_state
                            logger.info(
                                f"[监控循环] 实验{self.experiment_id}初始状态: {current_state}"
                            )
                        elif current_state != self._last_state:
                            # 状态发生变化
                            old_state = self._last_state
                            self._last_state = current_state
                            
                            logger.info(
                                f"[监控循环] 🔄 实验{self.experiment_id}状态变化: {old_state} -> {current_state}"
                            )
                            
                            # 处理状态变化
                            self._handle_state_change(old_state, current_state)
                        else:
                            logger.debug(
                                f"[监控循环] 实验{self.experiment_id}状态无变化: {current_state}"
                            )
                    else:
                        logger.debug(f"[监控循环] 实验{self.experiment_id}暂无数据")
                    
                    # 等待下一个轮询周期
                    self._stop_event.wait(self.poll_interval)
                    
                except Exception as e:
                    logger.error(f"[监控循环] 实验{self.experiment_id}查询错误: {e}", exc_info=True)
                    self._stop_event.wait(self.poll_interval)
        
        except Exception as e:
            logger.error(f"[监控循环] 实验{self.experiment_id}监控线程异常: {e}", exc_info=True)
        finally:
            logger.info(f"[监控循环] 实验{self.experiment_id}监控线程已退出")
    
    def _query_current_state(self) -> Optional[str]:
        """
        查询当前状态（带重试机制）
        
        Returns:
            当前状态值，如果查询失败返回None
        """
        # 查询配置验证
        bucket = self.query_config.get('bucket', '')
        measurement = self.query_config.get('measurement', '')
        fields = self.query_config.get('fields', [])
        filters = self.query_config.get('filters', {})
        status_field = self.query_config.get('status_field', '')
        
        if not all([bucket, measurement, fields, status_field]):
            logger.warning(
                f"[查询] 实验{self.experiment_id}查询配置不完整: "
                f"bucket={bucket}, measurement={measurement}, fields={fields}, status_field={status_field}"
            )
            return None
        
        # 带重试的查询
        for attempt in range(self.max_retry_attempts):
            try:
                # 执行查询
                result = self._execute_query(bucket, measurement, fields, filters, status_field)
                
                # 查询成功，更新连接状态
                self._on_query_success()
                
                return result
                
            except Exception as e:
                # 查询失败，记录错误
                is_last_attempt = (attempt == self.max_retry_attempts - 1)
                
                if is_last_attempt:
                    logger.error(
                        f"[查询] 实验{self.experiment_id}查询失败（已重试{attempt + 1}次）: {e}",
                        exc_info=True
                    )
                    # 更新失败状态
                    self._on_query_failure(str(e))
                else:
                    # 计算退避时间
                    backoff_time = self.retry_backoff_base ** attempt
                    logger.warning(
                        f"[查询] 实验{self.experiment_id}查询失败（第{attempt + 1}次尝试），"
                        f"{backoff_time:.1f}秒后重试: {e}"
                    )
                    # 等待后重试
                    time.sleep(backoff_time)
        
        return None
    
    def _execute_query(self, bucket: str, measurement: str, fields: list, 
                       filters: dict, status_field: str) -> Optional[str]:
        """
        执行单次查询（不含重试逻辑）
        
        Returns:
            当前状态值，如果查询结果为空返回None
        
        Raises:
            Exception: 查询过程中的任何异常
        """
        # 创建服务实例
        service = InfluxService(self.influx_params)
        
        # 构建时间范围：查询最近1小时的数据
        time_range = "-1h"
        
        logger.debug(
            f"[查询] 实验{self.experiment_id}查询InfluxDB: "
            f"bucket={bucket}, measurement={measurement}, fields={fields}"
        )
        
        # 执行查询
        df = service.query(
            bucket=bucket,
            measurement=measurement,
            fields=fields,
            filters=filters,
            time_range=time_range
        )
        
        if df.empty:
            logger.debug(f"[查询] 实验{self.experiment_id}查询结果为空")
            return None
        
        # 获取最新的状态值
        if '_value' in df.columns and '_field' in df.columns:
            # 筛选出指定字段的数据
            status_data = df[df['_field'] == status_field]
            
            if not status_data.empty:
                # 按时间排序，取最后一条记录
                if '_time' in status_data.columns:
                    status_data = status_data.sort_values('_time')
                
                latest_value = status_data['_value'].iloc[-1]
                logger.debug(
                    f"[查询] 实验{self.experiment_id}最新状态值: {latest_value} "
                    f"(字段: {status_field})"
                )
                return str(latest_value)
            else:
                logger.warning(
                    f"[查询] 实验{self.experiment_id}未找到字段'{status_field}'的数据"
                )
                return None
        else:
            logger.warning(
                f"[查询] 实验{self.experiment_id}查询结果格式异常, "
                f"可用字段: {list(df.columns)}"
            )
            return None
    
    def _on_query_success(self) -> None:
        """
        查询成功时的处理
        """
        with self._connection_lock:
            was_disconnected = not self._is_connected
            
            # 重置失败计数
            self._consecutive_failures = 0
            self._last_success_time = datetime.datetime.now()
            
            # 更新连接状态
            if was_disconnected:
                self._is_connected = True
                elapsed = None
                if self._last_success_time:
                    elapsed = (datetime.datetime.now() - self._last_success_time).total_seconds()
                
                message = f"实验{self.experiment_id}已恢复与InfluxDB的连接"
                logger.info(f"[连接恢复] ✅ {message}")
                
                # 触发连接状态变化回调
                if self.on_connection_changed:
                    try:
                        self.on_connection_changed(True, message)
                    except Exception as e:
                        logger.error(f"[连接回调] 执行连接状态回调失败: {e}", exc_info=True)
    
    def _on_query_failure(self, error_message: str) -> None:
        """
        查询失败时的处理
        
        Args:
            error_message: 错误信息
        """
        with self._connection_lock:
            self._consecutive_failures += 1
            
            # 判断是否应该标记为断开连接
            if self._consecutive_failures >= self.max_consecutive_failures:
                was_connected = self._is_connected
                
                if was_connected:
                    self._is_connected = False
                    message = (
                        f"实验{self.experiment_id}与InfluxDB连接断开 "
                        f"(连续失败{self._consecutive_failures}次): {error_message}"
                    )
                    logger.error(f"[连接断开] ❌ {message}")
                    
                    # 触发连接状态变化回调
                    if self.on_connection_changed:
                        try:
                            self.on_connection_changed(False, message)
                        except Exception as e:
                            logger.error(f"[连接回调] 执行连接状态回调失败: {e}", exc_info=True)
                else:
                    logger.warning(
                        f"[连接状态] 实验{self.experiment_id}仍处于断开状态 "
                        f"(连续失败{self._consecutive_failures}次)"
                    )
            else:
                logger.warning(
                    f"[连接状态] 实验{self.experiment_id}查询失败 "
                    f"(连续失败{self._consecutive_failures}/{self.max_consecutive_failures}次)"
                )
    
    def _handle_state_change(self, old_state: str, new_state: str) -> None:
        """
        处理状态变化
        
        Args:
            old_state: 旧状态
            new_state: 新状态
        """
        try:
            status_values = self.query_config.get('status_values', {})
            start_value = status_values.get('start', '')
            end_value = status_values.get('end', '')
            
            logger.info(
                f"[状态变化] 实验{self.experiment_id}: {old_state} -> {new_state}, "
                f"开始值={start_value}, 结束值={end_value}"
            )
            
            # 检测实验开始：状态从非开始值变为开始值
            if not self._experiment_started and str(new_state) == str(start_value):
                self._on_experiment_started()
            
            # 检测实验结束：状态从开始值变回非开始值
            elif self._experiment_started and str(new_state) == str(end_value):
                self._on_experiment_ended()
            
            # 触发回调
            if self.on_state_changed:
                self.on_state_changed(old_state, new_state)
        
        except Exception as e:
            logger.error(f"[状态变化] 实验{self.experiment_id}处理失败: {e}", exc_info=True)
    
    def _on_experiment_started(self) -> None:
        """实验开始事件处理"""
        try:
            self._experiment_started = True
            self._start_time_recorded = datetime.datetime.now().isoformat(timespec='seconds')
            
            logger.info(
                f"[实验开始] 🟢 实验{self.experiment_id}已开始, "
                f"记录时间: {self._start_time_recorded}"
            )
            
            # 更新数据库：设置start_ts
            self._update_experiment_start_time(self._start_time_recorded)
            
            logger.info(f"[实验开始] ✅ 实验{self.experiment_id}开始时间已记录并更新数据库")
        
        except Exception as e:
            logger.error(f"[实验开始] 实验{self.experiment_id}处理失败: {e}", exc_info=True)
    
    def _on_experiment_ended(self) -> None:
        """实验结束事件处理"""
        try:
            # 检查实验是否处于暂停状态
            if self._is_experiment_paused():
                logger.info(
                    f"[实验结束] ⏸️ 实验{self.experiment_id}处于暂停状态，"
                    f"忽略状态变化，不记录结束时间"
                )
                return
            
            self._experiment_ended = True
            self._end_time_recorded = datetime.datetime.now().isoformat(timespec='seconds')
            
            logger.info(
                f"[实验结束] 🔴 实验{self.experiment_id}已结束, "
                f"记录时间: {self._end_time_recorded}"
            )
            
            # 更新数据库：设置end_ts
            self._update_experiment_end_time(self._end_time_recorded)
            
            logger.info(f"[实验结束] ✅ 实验{self.experiment_id}结束时间已记录并更新数据库")
            
            # 设置停止标志，让监控循环自然退出
            # 不在这里直接调用stop()，避免线程join自己的问题
            self._stop_event.set()
            logger.info(f"[实验结束] 🛑 已设置停止标志，监控将自动退出")
        
        except Exception as e:
            logger.error(f"[实验结束] 实验{self.experiment_id}处理失败: {e}", exc_info=True)
    
    def _is_experiment_paused(self) -> bool:
        """检查实验是否处于暂停状态"""
        try:
            from pathlib import Path
            db_path = Path(__file__).parent / "experiments.db"
            
            db = sqlite3.connect(str(db_path))
            cur = db.cursor()
            
            cur.execute(
                "SELECT is_paused FROM experiments WHERE id=?",
                (self.experiment_id,)
            )
            result = cur.fetchone()
            db.close()
            
            if result:
                is_paused = result[0] == 1
                logger.debug(f"[暂停检查] 实验{self.experiment_id}暂停状态: {is_paused}")
                return is_paused
            else:
                logger.warning(f"[暂停检查] 实验{self.experiment_id}未找到记录")
                return False
        
        except Exception as e:
            logger.error(
                f"[暂停检查] 实验{self.experiment_id}查询失败: {e}",
                exc_info=True
            )
            return False
    
    def _update_experiment_start_time(self, start_time: str) -> None:
        """更新实验记录的开始时间"""
        try:
            from pathlib import Path
            db_path = Path(__file__).parent / "experiments.db"
            
            db = sqlite3.connect(str(db_path))
            cur = db.cursor()
            
            cur.execute(
                "UPDATE experiments SET start_ts=? WHERE id=?",
                (start_time, self.experiment_id)
            )
            db.commit()
            db.close()
            
            logger.info(
                f"[数据库更新] 实验{self.experiment_id}开始时间已更新: {start_time}"
            )
        
        except Exception as e:
            logger.error(
                f"[数据库更新] 实验{self.experiment_id}更新开始时间失败: {e}",
                exc_info=True
            )
    
    def _update_experiment_end_time(self, end_time: str) -> None:
        """更新实验记录的结束时间，并执行动态脚本"""
        try:
            from pathlib import Path
            db_path = Path(__file__).parent / "experiments.db"
            
            db = sqlite3.connect(str(db_path))
            cur = db.cursor()
            
            cur.execute(
                "UPDATE experiments SET end_ts=? WHERE id=?",
                (end_time, self.experiment_id)
            )
            db.commit()
            db.close()
            
            logger.info(
                f"[数据库更新] 实验{self.experiment_id}结束时间已更新: {end_time}"
            )
            
            # 实验结束后，自动执行动态脚本并保存数据
            self._execute_and_save_script_data()
        
        except Exception as e:
            logger.error(
                f"[数据库更新] 实验{self.experiment_id}更新结束时间失败: {e}",
                exc_info=True
            )
    
    def _execute_and_save_script_data(self) -> None:
        """执行动态脚本并保存返回数据到数据库"""
        try:
            from pathlib import Path
            import json
            
            logger.info(f"[脚本执行] 开始为实验{self.experiment_id}执行动态脚本")
            
            # 获取实验配置
            db_path = Path(__file__).parent / "experiments.db"
            db = sqlite3.connect(str(db_path))
            cur = db.cursor()
            
            cur.execute(
                "SELECT config_json, work_order_no FROM experiments WHERE id=?",
                (self.experiment_id,)
            )
            result = cur.fetchone()
            
            if not result:
                logger.warning(f"[脚本执行] 实验{self.experiment_id}未找到配置")
                db.close()
                return
            
            config_json, work_order_no = result
            db.close()
            
            # 解析配置
            from config_model import AppConfig
            from tempfile import NamedTemporaryFile
            
            with NamedTemporaryFile('w', delete=False, suffix='.json', encoding='utf-8') as tf:
                tf.write(config_json)
                snap_path = Path(tf.name)
            
            config = AppConfig.load(snap_path)
            
            # 执行脚本
            from report_generator import _execute_experiment_script
            script_data = _execute_experiment_script(config)
            
            if script_data:
                # 保存脚本数据到 SQLite 数据库
                script_data_json = json.dumps(script_data, ensure_ascii=False)
                
                db = sqlite3.connect(str(db_path))
                cur = db.cursor()
                
                cur.execute(
                    "UPDATE experiments SET script_data=? WHERE id=?",
                    (script_data_json, self.experiment_id)
                )
                db.commit()
                db.close()
                
                logger.info(
                    f"[脚本执行] ✅ 实验{self.experiment_id}脚本数据已保存到 SQLite "
                    f"(数据大小: {len(script_data_json)} 字节)"
                )
                
                # 写入 SQL Server（如果配置了）
                self._write_to_sqlserver(script_data, work_order_no, config)
            else:
                logger.warning(f"[脚本执行] 实验{self.experiment_id}脚本未返回数据")
        
        except Exception as e:
            logger.error(
                f"[脚本执行] 实验{self.experiment_id}执行脚本失败: {e}",
                exc_info=True
            )
    
    def _write_to_sqlserver(self, script_data: dict, work_order_no: str, config) -> None:
        """
        将脚本数据写入 SQL Server
        
        Args:
            script_data: 脚本返回的数据
            work_order_no: 工单号
            config: 应用配置
        """
        try:
            # 从 work_order_db_config.json 加载 SQL Server 配置
            from pathlib import Path
            import json
            
            config_file = Path(__file__).parent / 'work_order_db_config.json'
            
            if not config_file.exists():
                logger.info(f"[SQL Server] 配置文件不存在，跳过写入: {config_file}")
                return
            
            # 读取配置
            with open(config_file, 'r', encoding='utf-8') as f:
                db_config = json.load(f)
            
            # 检查是否为调试模式
            if db_config.get('debug_mode', False):
                logger.info(f"[SQL Server] 调试模式已启用，跳过实际写入")
                return
            
            sqlserver_config = {
                'host': db_config.get('host', 'localhost'),
                'port': db_config.get('port', 1433),
                'database': db_config.get('database', ''),
                'username': db_config.get('username', ''),
                'password': db_config.get('password', ''),
            }
            
            # 检查必要配置
            if not sqlserver_config['database']:
                logger.warning(f"[SQL Server] 数据库名未配置，跳过写入")
                return
            
            logger.info(f"[SQL Server] 开始写入实验{self.experiment_id}的数据")
            
            # 准备写入数据
            from sqlserver_writer import write_script_data_to_sqlserver
            from convert_table_to_sqlserver_format import convert_temperature_table_to_sqlserver
            
            # 获取全局参数
            global_params = {}
            if hasattr(config, 'globalParameters') and config.globalParameters:
                global_params = config.globalParameters.parameters or {}
            
            # 转换表格数据为 SQL Server 格式
            write_data = convert_temperature_table_to_sqlserver(script_data, work_order_no, global_params)
            
            if not write_data:
                logger.warning(f"[SQL Server] 数据转换失败，跳过写入")
                return
            
            # 写入数据
            success = write_script_data_to_sqlserver(write_data, sqlserver_config)
            
            if success:
                logger.info(f"[SQL Server] ✅ 实验{self.experiment_id}数据已写入 SQL Server")
            else:
                logger.warning(f"[SQL Server] ⚠️ 实验{self.experiment_id}数据写入 SQL Server 失败")
        
        except Exception as e:
            logger.error(
                f"[SQL Server] 实验{self.experiment_id}写入 SQL Server 异常: {e}",
                exc_info=True
            )
    
    def get_status(self) -> Dict[str, Any]:
        """获取监控器状态"""
        with self._connection_lock:
            return {
                'experiment_id': self.experiment_id,
                'work_order_no': self.work_order_no,
                'is_running': self._thread is not None and self._thread.is_alive(),
                'is_connected': self._is_connected,
                'consecutive_failures': self._consecutive_failures,
                'last_success_time': self._last_success_time.isoformat() if self._last_success_time else None,
                'last_state': self._last_state,
                'experiment_started': self._experiment_started,
                'experiment_ended': self._experiment_ended,
                'start_time_recorded': self._start_time_recorded,
                'end_time_recorded': self._end_time_recorded
            }