DataOps-platform/scripts/deploy_dataops.sh

#!/bin/bash
#
# DataOps Platform 部署脚本
# 用于初始化或重建虚拟环境并配置 supervisor
#

set -e

# 配置变量
APP_NAME="dataops-platform"
APP_DIR="/opt/dataops-platform"
VENV_DIR="${APP_DIR}/venv"
LOG_DIR="${APP_DIR}/logs"
PYTHON_VERSION="python3"
SUPERVISOR_CONF="/etc/supervisor/conf.d/${APP_NAME}.conf"

# Gunicorn 配置
GUNICORN_WORKERS=4
GUNICORN_BIND="0.0.0.0:80"
GUNICORN_TIMEOUT=120

# 颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

echo_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

echo_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

echo_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

echo_step() {
    echo -e "${BLUE}[STEP]${NC} $1"
}

# 检查是否以 root 或 sudo 运行
check_permissions() {
    if [ "$EUID" -ne 0 ]; then
        echo_error "请使用 sudo 运行此脚本"
        exit 1
    fi
}

# 检查 Python 版本
check_python() {
    echo_step "检查 Python 版本..."
    
    if ! command -v ${PYTHON_VERSION} &> /dev/null; then
        echo_error "Python3 未安装"
        exit 1
    fi
    
    python_ver=$(${PYTHON_VERSION} --version 2>&1 | awk '{print $2}')
    echo_info "Python 版本: ${python_ver}"
    
    # 检查 python3-venv
    if ! dpkg -l | grep -q python3-venv; then
        echo_warn "python3-venv 未安装，正在安装..."
        apt-get update && apt-get install -y python3-venv python3-pip
    fi
}

# 检查 supervisor
check_supervisor() {
    echo_step "检查 Supervisor..."
    
    if ! command -v supervisord &> /dev/null; then
        echo_warn "Supervisor 未安装，正在安装..."
        apt-get update && apt-get install -y supervisor
        systemctl enable supervisor
        systemctl start supervisor
    fi
    
    echo_info "Supervisor 已安装"
}

# 创建目录结构
create_directories() {
    echo_step "创建目录结构..."
    
    mkdir -p ${LOG_DIR}
    chown -R ubuntu:ubuntu ${LOG_DIR}
    
    echo_info "日志目录: ${LOG_DIR}"
}

# 创建虚拟环境
create_venv() {
    echo_step "创建虚拟环境..."
    
    # 如果虚拟环境存在，先备份再删除
    if [ -d "${VENV_DIR}" ]; then
        echo_warn "发现已存在的虚拟环境，正在删除..."
        rm -rf ${VENV_DIR}
    fi
    
    # 创建新的虚拟环境
    ${PYTHON_VERSION} -m venv ${VENV_DIR}
    
    # 更新 pip
    ${VENV_DIR}/bin/pip install --upgrade pip
    
    echo_info "虚拟环境创建完成: ${VENV_DIR}"
}

# 安装依赖
install_dependencies() {
    echo_step "安装 Python 依赖..."
    
    if [ ! -f "${APP_DIR}/requirements.txt" ]; then
        echo_error "requirements.txt 不存在"
        exit 1
    fi
    
    # 安装依赖
    ${VENV_DIR}/bin/pip install -r ${APP_DIR}/requirements.txt
    
    # 确保 gunicorn 已安装
    ${VENV_DIR}/bin/pip install gunicorn
    
    echo_info "依赖安装完成"
    
    # 显示已安装的关键包
    echo_info "已安装的关键包:"
    ${VENV_DIR}/bin/pip list | grep -E "Flask|gunicorn|neo4j|SQLAlchemy|psycopg2"
}

# 验证安装
verify_installation() {
    echo_step "验证安装..."
    
    # 测试导入
    cd ${APP_DIR}
    ${VENV_DIR}/bin/python -c "
from app import create_app
app = create_app()
print('Flask 应用创建成功')
print(f'已注册的蓝图: {list(app.blueprints.keys())}')
" || {
        echo_error "应用验证失败"
        exit 1
    }
    
    echo_info "应用验证通过"
}

# 创建 Gunicorn 配置文件
create_gunicorn_config() {
    echo_step "创建 Gunicorn 配置..."
    
    cat > ${APP_DIR}/gunicorn.conf.py << EOF
# Gunicorn 配置文件
import multiprocessing

# 绑定地址
bind = "${GUNICORN_BIND}"

# Worker 进程数
workers = ${GUNICORN_WORKERS}

# Worker 类型
worker_class = "sync"

# 超时时间
timeout = ${GUNICORN_TIMEOUT}

# 优雅重启超时
graceful_timeout = 30

# 保持连接时间
keepalive = 5

# 最大请求数（防止内存泄漏）
max_requests = 1000
max_requests_jitter = 50

# 日志配置
accesslog = "${LOG_DIR}/gunicorn_access.log"
errorlog = "${LOG_DIR}/gunicorn_error.log"
loglevel = "info"

# 进程名
proc_name = "${APP_NAME}"

# 工作目录
chdir = "${APP_DIR}"

# 预加载应用
preload_app = True

# 环境变量
raw_env = [
    "FLASK_ENV=production",
]
EOF

    chown ubuntu:ubuntu ${APP_DIR}/gunicorn.conf.py
    echo_info "Gunicorn 配置文件已创建: ${APP_DIR}/gunicorn.conf.py"
}

# 创建 WSGI 入口文件
create_wsgi() {
    echo_step "创建 WSGI 入口文件..."
    
    cat > ${APP_DIR}/wsgi.py << 'EOF'
"""
WSGI 入口文件
用于 Gunicorn 启动 Flask 应用
"""
from app import create_app

application = create_app()

if __name__ == "__main__":
    application.run()
EOF

    chown ubuntu:ubuntu ${APP_DIR}/wsgi.py
    echo_info "WSGI 入口文件已创建: ${APP_DIR}/wsgi.py"
}

# 配置 Supervisor
configure_supervisor() {
    echo_step "配置 Supervisor..."
    
    cat > ${SUPERVISOR_CONF} << EOF
[program:${APP_NAME}]
command=${VENV_DIR}/bin/gunicorn -c ${APP_DIR}/gunicorn.conf.py wsgi:application
directory=${APP_DIR}
user=ubuntu
autostart=true
autorestart=true
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=${LOG_DIR}/supervisor_stdout.log
stderr_logfile=${LOG_DIR}/supervisor_stderr.log
environment=FLASK_ENV="production",PATH="${VENV_DIR}/bin:%(ENV_PATH)s"
EOF

    echo_info "Supervisor 配置已创建: ${SUPERVISOR_CONF}"
    
    # 重新加载 supervisor 配置
    supervisorctl reread
    supervisorctl update
    
    echo_info "Supervisor 配置已更新"
}

# 设置文件权限
set_permissions() {
    echo_step "设置文件权限..."
    
    chown -R ubuntu:ubuntu ${APP_DIR}
    chmod +x ${APP_DIR}/scripts/*.sh 2>/dev/null || true
    
    echo_info "文件权限设置完成"
}

# 启动应用
start_application() {
    echo_step "启动应用..."
    
    supervisorctl start ${APP_NAME}
    
    sleep 3
    
    status=$(supervisorctl status ${APP_NAME} | awk '{print $2}')
    if [ "$status" = "RUNNING" ]; then
        echo_info "应用启动成功!"
        supervisorctl status ${APP_NAME}
    else
        echo_error "应用启动失败，请检查日志"
        echo_info "查看日志: tail -f ${LOG_DIR}/gunicorn_error.log"
        exit 1
    fi
}

# 健康检查
health_check() {
    echo_step "执行健康检查..."
    
    local max_retries=5
    local retry_interval=3
    local retry_count=0
    local response=""
    local APP_PORT=5500
    
    while [ $retry_count -lt $max_retries ]; do
        sleep $retry_interval
        retry_count=$((retry_count + 1))
        
        # 尝试健康检查接口（使用应用实际端口 5500）
        response=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:${APP_PORT}/api/system/health 2>/dev/null || echo "000")
        
        if [ "$response" = "200" ]; then
            echo_info "健康检查通过! HTTP 状态码: ${response}"
            return 0
        fi
        
        echo_info "尝试 ${retry_count}/${max_retries}: HTTP 状态码 ${response}，等待重试..."
    done
    
    # 如果 /api/system/health 失败，尝试其他接口作为备选
    echo_warn "健康检查接口返回状态码: ${response}"
    
    # 尝试检查 /api/bd/list 接口作为备选（使用 POST 方法）
    response=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://127.0.0.1:${APP_PORT}/api/bd/list -H "Content-Type: application/json" -d "{}" 2>/dev/null || echo "000")
    if [ "$response" = "200" ] || [ "$response" = "500" ]; then
        echo_info "备选接口 /api/bd/list 有响应（${response}），服务已启动!"
        return 0
    fi
    
    echo_warn "服务可能需要更多时间启动，或健康检查接口配置有问题"
    echo_info "请手动检查: curl http://127.0.0.1:${APP_PORT}/api/system/health"
}

# 显示部署信息
show_summary() {
    echo ""
    echo "=========================================="
    echo -e "${GREEN}  部署完成!${NC}"
    echo "=========================================="
    echo ""
    echo "应用信息:"
    echo "  - 应用名称: ${APP_NAME}"
    echo "  - 应用目录: ${APP_DIR}"
    echo "  - 虚拟环境: ${VENV_DIR}"
    echo "  - 日志目录: ${LOG_DIR}"
    echo "  - 监听地址: ${GUNICORN_BIND}"
    echo ""
    echo "常用命令:"
    echo "  - 启动: sudo ${APP_DIR}/scripts/start_dataops.sh"
    echo "  - 停止: sudo ${APP_DIR}/scripts/stop_dataops.sh"
    echo "  - 重启: sudo ${APP_DIR}/scripts/restart_dataops.sh"
    echo "  - 状态: sudo supervisorctl status ${APP_NAME}"
    echo "  - 日志: tail -f ${LOG_DIR}/gunicorn_error.log"
    echo ""
    echo "Supervisor 命令:"
    echo "  - sudo supervisorctl status"
    echo "  - sudo supervisorctl restart ${APP_NAME}"
    echo "  - sudo supervisorctl tail -f ${APP_NAME}"
    echo ""
}

# 主函数
main() {
    echo "=========================================="
    echo "  DataOps Platform 部署脚本"
    echo "=========================================="
    echo ""
    
    check_permissions
    check_python
    check_supervisor
    create_directories
    create_venv
    install_dependencies
    verify_installation
    create_gunicorn_config
    create_wsgi
    configure_supervisor
    set_permissions
    start_application
    health_check
    show_summary
}

main "$@"