DataOps-platform/deployment/deploy_dataops.sh

#!/usr/bin/env bash
#
# DataOps Platform 一键部署脚本
# 在 deployment/ 目录（或解压后的发布包根目录）内执行
#
# 用法:
#   sudo bash deploy_dataops.sh
#   sudo ENABLE_NGINX=1 NGINX_SERVER_NAME=company.citupro.com NGINX_LISTEN_PORT=18183 bash deploy_dataops.sh
#
set -euo pipefail

APP_NAME="${APP_NAME:-dataops-platform}"
APP_DIR="${APP_DIR:-/opt/dataops-platform}"
APP_USER="${APP_USER:-ubuntu}"
APP_GROUP="${APP_GROUP:-$APP_USER}"
ENV_DIR="${ENV_DIR:-/etc/dataops-platform}"
ENV_FILE="${ENV_FILE:-$ENV_DIR/dataops.env}"
LISTEN_HOST="${LISTEN_HOST:-0.0.0.0}"
LISTEN_PORT="${LISTEN_PORT:-5500}"
GUNICORN_WORKERS="${GUNICORN_WORKERS:-4}"
GUNICORN_TIMEOUT="${GUNICORN_TIMEOUT:-120}"
ENABLE_NGINX="${ENABLE_NGINX:-0}"
NGINX_SERVER_NAME="${NGINX_SERVER_NAME:-company.citupro.com}"
NGINX_LISTEN_PORT="${NGINX_LISTEN_PORT:-18183}"
NGINX_SSL="${NGINX_SSL:-1}"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SUDO=""
if [[ "$(id -u)" -ne 0 ]]; then
  SUDO="sudo"
fi

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
info() { echo -e "${GREEN}[INFO]${NC} $*"; }
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
err() { echo -e "${RED}[ERROR]${NC} $*"; }
step() { echo -e "${BLUE}[STEP]${NC} $*"; }

require_command() {
  if ! command -v "$1" >/dev/null 2>&1; then
    err "缺少命令: $1"
    exit 1
  fi
}

normalize_text_files() {
  local path
  for path in "$@"; do
    [[ -e "${path}" ]] || continue
    if [[ -d "${path}" ]]; then
      find "${path}" -type f \( -name '*.sh' -o -name '*.env' -o -name '.env*' \) -print0 2>/dev/null \
        | while IFS= read -r -d '' file; do sed -i 's/\r$//' "${file}" 2>/dev/null || true; done
    elif [[ -f "${path}" ]]; then
      sed -i 's/\r$//' "${path}" 2>/dev/null || true
    fi
  done
}

require_command python3
require_command supervisorctl

for item in app requirements.txt wsgi.py gunicorn_config.py scripts/run_dataops.sh; do
  if [[ ! -e "${SCRIPT_DIR}/${item}" ]]; then
    err "部署包不完整，缺少: ${SCRIPT_DIR}/${item}"
    err "请确认在 deployment/ 或发布包根目录内执行本脚本"
    exit 1
  fi
done

install_env_file() {
  step "配置环境变量 ${ENV_FILE}"
  $SUDO install -d -m 750 "${ENV_DIR}"

  if [[ -f "${ENV_FILE}" ]]; then
    info "使用已有环境变量: ${ENV_FILE}"
    normalize_text_files "${ENV_FILE}"
    return 0
  fi

  if [[ -f "${SCRIPT_DIR}/dataops.env" ]]; then
    $SUDO install -m 640 "${SCRIPT_DIR}/dataops.env" "${ENV_FILE}"
    info "已从 dataops.env 安装到 ${ENV_FILE}"
  elif [[ -f "${SCRIPT_DIR}/.env.production.example" ]]; then
    $SUDO install -m 640 "${SCRIPT_DIR}/.env.production.example" "${ENV_FILE}"
    info "已从 .env.production.example 安装到 ${ENV_FILE}"
  else
    err "未找到 dataops.env 或 .env.production.example"
    exit 1
  fi

  $SUDO chown root:"${APP_GROUP}" "${ENV_FILE}" 2>/dev/null || true
  normalize_text_files "${ENV_FILE}"
  warn "首次部署：请编辑 ${ENV_FILE} 填入真实密钥后重新执行本脚本"
  exit 2
}

install_env_file

DATABASE_ENV_VALIDATOR="${SCRIPT_DIR}/app/config/database_urls.py"
if [[ ! -f "${DATABASE_ENV_VALIDATOR}" ]] || \
   ! python3 "${DATABASE_ENV_VALIDATOR}" --env-file "${ENV_FILE}"; then
  warn "数据库连接或 runtime 密码仍为空、格式错误或包含模板占位值"
  exit 2
fi

if grep -qE 'replace-with|replace-n8n|your-api-key|your-deepseek' "${ENV_FILE}" 2>/dev/null; then
  warn "环境变量仍含模板占位符，请编辑 ${ENV_FILE} 后重新部署"
  warn "必须配置: SECRET_KEY, DEEPSEEK_API_KEY, N8N_API_KEY 及数据库/MinIO 连接"
  exit 2
fi

step "创建目录"
$SUDO install -d -m 755 "${APP_DIR}" "${APP_DIR}/logs" /data/upload /data/archive /var/log/supervisor

step "发布应用代码到 ${APP_DIR}"
for target in app database; do
  $SUDO rm -rf "${APP_DIR}/${target}"
  $SUDO cp -a "${SCRIPT_DIR}/${target}" "${APP_DIR}/${target}"
done
for file in requirements.txt wsgi.py gunicorn_config.py __init__.py; do
  if [[ -f "${SCRIPT_DIR}/${file}" ]]; then
    $SUDO cp -f "${SCRIPT_DIR}/${file}" "${APP_DIR}/${file}"
  fi
done

step "发布运维脚本到 ${APP_DIR}/scripts"
OPS_SRC="${SCRIPT_DIR}/scripts"
if [[ ! -d "${OPS_SRC}" ]]; then
  err "缺少 scripts/ 目录"
  exit 1
fi
$SUDO rm -rf "${APP_DIR}/scripts"
$SUDO cp -a "${OPS_SRC}" "${APP_DIR}/scripts"
$SUDO chmod +x "${APP_DIR}/scripts/"*.sh
$SUDO rm -f "${APP_DIR}/run_dataops.sh" 2>/dev/null || true
normalize_text_files "${APP_DIR}/scripts"

step "创建 Python 虚拟环境并安装依赖"
if [[ ! -d "${APP_DIR}/venv" ]]; then
  $SUDO -u "${APP_USER}" python3 -m venv "${APP_DIR}/venv"
fi
$SUDO "${APP_DIR}/venv/bin/python" -m pip install --upgrade pip
$SUDO "${APP_DIR}/venv/bin/pip" install -r "${APP_DIR}/requirements.txt"

step "验证 Flask 应用"
$SUDO -u "${APP_USER}" env APP_ENV_FILE="${ENV_FILE}" bash -c "
  set -a
  source <(sed 's/\r$//' '${ENV_FILE}' | sed '1s/^\xEF\xBB\xBF//')
  set +a
  cd '${APP_DIR}'
  '${APP_DIR}/venv/bin/python' -c \"from app import create_app; create_app(); print('Flask 应用验证通过')\"
" || {
  err "应用验证失败，请检查 ${ENV_FILE}"
  exit 1
}

step "配置 Supervisor"
if [[ -f "${APP_DIR}/gunicorn.conf.py" ]]; then
  $SUDO mv "${APP_DIR}/gunicorn.conf.py" "${APP_DIR}/gunicorn.conf.py.bak.$(date +%Y%m%d%H%M%S)" 2>/dev/null || true
  warn "已备份旧 gunicorn.conf.py"
fi
$SUDO tee "/etc/supervisor/conf.d/${APP_NAME}.conf" >/dev/null <<EOF
[program:${APP_NAME}]
command=/usr/bin/env bash ${APP_DIR}/scripts/run_dataops.sh
directory=${APP_DIR}
user=${APP_USER}
autostart=true
autorestart=true
startsecs=8
startretries=5
stopasgroup=true
killasgroup=true
redirect_stderr=true
stdout_logfile=/var/log/supervisor/${APP_NAME}.log
stdout_logfile_maxbytes=50MB
stdout_logfile_backups=5
environment=FLASK_ENV="production",APP_ENV_FILE="${ENV_FILE}",APP_DIR="${APP_DIR}",PATH="${APP_DIR}/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
EOF

step "设置权限"
$SUDO chown -R "${APP_USER}:${APP_GROUP}" "${APP_DIR}" /data/upload /data/archive
$SUDO chown root:"${APP_GROUP}" "${ENV_FILE}" 2>/dev/null || true
$SUDO chmod 640 "${ENV_FILE}" 2>/dev/null || $SUDO chmod 600 "${ENV_FILE}"

step "启动服务"
$SUDO supervisorctl reread
$SUDO supervisorctl update
$SUDO supervisorctl restart "${APP_NAME}" 2>/dev/null || $SUDO supervisorctl start "${APP_NAME}"
sleep 8

health_ok=0
for i in $(seq 1 8); do
  code="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${LISTEN_PORT}/api/system/health" 2>/dev/null || echo 000)"
  if [[ "${code}" == "200" ]]; then
    health_ok=1
    break
  fi
  info "健康检查 ${i}/8: HTTP ${code}，等待..."
  sleep 3
done

if [[ "${ENABLE_NGINX}" == "1" ]]; then
  require_command nginx
  step "配置 Nginx 反向代理"
  NGINX_CONF="/etc/nginx/sites-available/${APP_NAME}.conf"
  if [[ -f "${SCRIPT_DIR}/config/nginx-dataops-platform.conf" ]]; then
    $SUDO cp -f "${SCRIPT_DIR}/config/nginx-dataops-platform.conf" "${NGINX_CONF}"
    $SUDO sed -i "s/company.example.com/${NGINX_SERVER_NAME}/g" "${NGINX_CONF}"
    $SUDO sed -i "s/listen 18183/listen ${NGINX_LISTEN_PORT}/g" "${NGINX_CONF}"
    $SUDO sed -i "s/127.0.0.1:5500/127.0.0.1:${LISTEN_PORT}/g" "${NGINX_CONF}"
  else
    $SUDO tee "${NGINX_CONF}" >/dev/null <<EOF
server {
    listen ${NGINX_LISTEN_PORT};
    server_name ${NGINX_SERVER_NAME};
    client_max_body_size 100m;
    location /api/bd/ddlparse {
        proxy_pass http://127.0.0.1:${LISTEN_PORT};
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_connect_timeout 60s;
        proxy_send_timeout 300s;
        proxy_read_timeout 300s;
    }
    location / {
        proxy_pass http://127.0.0.1:${LISTEN_PORT};
        proxy_http_version 1.1;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
        proxy_connect_timeout 60s;
        proxy_send_timeout 180s;
        proxy_read_timeout 180s;
    }
}
EOF
  fi
  $SUDO ln -sfn "${NGINX_CONF}" "/etc/nginx/sites-enabled/${APP_NAME}.conf"
  $SUDO nginx -t
  $SUDO systemctl reload nginx
fi

echo ""
echo "=========================================="
echo -e "${GREEN}  部署完成${NC}"
echo "=========================================="
echo "  应用目录: ${APP_DIR}"
echo "  环境变量: ${ENV_FILE}"
echo "  监听地址: ${LISTEN_HOST}:${LISTEN_PORT}"
echo ""
echo "  启动: sudo ${APP_DIR}/scripts/start_dataops.sh"
echo "  停止: sudo ${APP_DIR}/scripts/stop_dataops.sh"
echo "  重启: sudo ${APP_DIR}/scripts/restart_dataops.sh"
echo "  健康: curl http://127.0.0.1:${LISTEN_PORT}/api/system/health"
echo ""

$SUDO supervisorctl status "${APP_NAME}" || true

if [[ "${health_ok}" -eq 1 ]]; then
  info "健康检查通过"
else
  warn "健康检查未通过，请查看: tail -f /var/log/supervisor/${APP_NAME}.log"
  exit 1
fi