483 lines
17 KiB
Bash
483 lines
17 KiB
Bash
#!/usr/bin/env bash
|
|
###############################################################################
|
|
# PaperClip — Master Deploy Script
|
|
# Fully idempotent. Safe to re-run at any time.
|
|
# Usage: sudo bash /opt/PaperClip/deploy_script.sh
|
|
###############################################################################
|
|
set -euo pipefail
|
|
|
|
REPO_DIR="/opt/PaperClip"
|
|
GIT_REPO_URL="https://github.com/orfelorfel23/PaperClip.git"
|
|
DATA_BASE="/opt/PaperClip-Data"
|
|
DOCKER_NETWORK="paperclip-net"
|
|
SERVICES_DIR="${REPO_DIR}/services"
|
|
AGE_PRIVATE_KEY="${REPO_DIR}/keys/age-key.txt"
|
|
CADDY_FRAGMENTS_DIR="${DATA_BASE}/caddy/fragments"
|
|
POSTGRES_CONTAINER="postgres"
|
|
|
|
# Infrastructure services that are started in a specific order
|
|
INFRA_SERVICES=("caddy")
|
|
|
|
ACTIVE_SERVICES=()
|
|
SKIPPED_SERVICES=()
|
|
|
|
NTFY_URL="https://ntfy.orfel.de/PaperClip"
|
|
|
|
# Send a push notification via ntfy (non-blocking, fails silently)
|
|
notify() {
|
|
local title="$1"
|
|
local message="$2"
|
|
local priority="${3:-default}"
|
|
local tags="${4:-}"
|
|
curl -s -o /dev/null -w "" \
|
|
-H "Title: ${title}" \
|
|
-H "Priority: ${priority}" \
|
|
-H "Tags: ${tags}" \
|
|
-d "${message}" \
|
|
"${NTFY_URL}" 2>/dev/null || true
|
|
}
|
|
|
|
###############################################################################
|
|
# Helpers
|
|
###############################################################################
|
|
log() { echo -e "\e[32m[DEPLOY]\e[0m $*"; }
|
|
warn() { echo -e "\e[33m[WARN]\e[0m $*"; }
|
|
err() { echo -e "\e[31m[ERROR]\e[0m $*" >&2; }
|
|
die() { err "$@"; exit 1; }
|
|
|
|
on_deploy_failure() {
|
|
err "Deployment GEFEHLT. Prüfe die Server-Logs."
|
|
notify "PaperClip Deploy FEHLGESCHLAGEN" "Deployment von PaperClip ist fehlgeschlagen. Prüfe die Server-Logs." "urgent" "x,rotating_light"
|
|
disable_maintenance_mode 2>/dev/null || true
|
|
}
|
|
|
|
is_active_service() {
|
|
local svc_dir="$1"
|
|
[[ -f "${svc_dir}/docker-compose.yml" && -f "${svc_dir}/service.enabled" ]]
|
|
}
|
|
|
|
###############################################################################
|
|
# 1. Install missing packages
|
|
###############################################################################
|
|
install_packages() {
|
|
log "Checking required packages..."
|
|
|
|
# System Updates
|
|
log "Running system updates..."
|
|
apt-get update -qq
|
|
DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -qq
|
|
|
|
# Docker
|
|
if ! command -v docker &>/dev/null; then
|
|
log "Installing Docker..."
|
|
apt-get update -qq
|
|
apt-get install -y -qq ca-certificates curl gnupg lsb-release
|
|
install -m 0755 -d /etc/apt/keyrings
|
|
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \
|
|
gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
|
chmod a+r /etc/apt/keyrings/docker.gpg
|
|
echo \
|
|
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
|
|
https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | \
|
|
tee /etc/apt/sources.list.d/docker.list > /dev/null
|
|
apt-get update -qq
|
|
apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
|
systemctl enable --now docker
|
|
else
|
|
log "Docker already installed."
|
|
fi
|
|
|
|
# age
|
|
if ! command -v age &>/dev/null; then
|
|
log "Installing age..."
|
|
apt-get update -qq
|
|
apt-get install -y -qq age
|
|
else
|
|
log "age already installed."
|
|
fi
|
|
|
|
# fail2ban
|
|
if ! command -v fail2ban-client &>/dev/null; then
|
|
log "Installing fail2ban..."
|
|
apt-get update -qq
|
|
apt-get install -y -qq fail2ban
|
|
else
|
|
log "fail2ban already installed."
|
|
fi
|
|
}
|
|
|
|
###############################################################################
|
|
# 2. Configure System (Docker, fail2ban, swap, timezone, cron)
|
|
###############################################################################
|
|
configure_system() {
|
|
log "Configuring Docker daemon log limits..."
|
|
mkdir -p /etc/docker
|
|
if ! grep -q "max-file" /etc/docker/daemon.json 2>/dev/null; then
|
|
cat > /etc/docker/daemon.json <<'EOF'
|
|
{
|
|
"log-driver": "json-file",
|
|
"log-opts": {
|
|
"max-size": "50m",
|
|
"max-file": "3"
|
|
}
|
|
}
|
|
EOF
|
|
systemctl restart docker || true
|
|
fi
|
|
|
|
log "Configuring fail2ban for SSH..."
|
|
cat > /etc/fail2ban/jail.local <<'EOF'
|
|
[sshd]
|
|
enabled = true
|
|
port = ssh
|
|
filter = sshd
|
|
logpath = /var/log/auth.log
|
|
maxretry = 5
|
|
bantime = 3600
|
|
findtime = 600
|
|
EOF
|
|
systemctl enable --now fail2ban
|
|
systemctl restart fail2ban
|
|
|
|
local SWAP_FILE="/swapfile"
|
|
local SWAP_SIZE="4G"
|
|
if ! swapon --show | grep -q "${SWAP_FILE}"; then
|
|
log "Setting up ${SWAP_SIZE} swap file..."
|
|
fallocate -l "${SWAP_SIZE}" "${SWAP_FILE}"
|
|
chmod 600 "${SWAP_FILE}"
|
|
mkswap "${SWAP_FILE}"
|
|
swapon "${SWAP_FILE}"
|
|
if ! grep -q "${SWAP_FILE}" /etc/fstab; then
|
|
echo "${SWAP_FILE} none swap sw 0 0" >> /etc/fstab
|
|
fi
|
|
fi
|
|
|
|
local TARGET_TZ="Europe/Berlin"
|
|
if ! timedatectl status | grep -q "${TARGET_TZ}"; then
|
|
timedatectl set-timezone "${TARGET_TZ}"
|
|
fi
|
|
|
|
log "Configuring weekly automatic reboot (Monday 1:23) and @reboot deployment..."
|
|
local CRON_REBOOT="23 1 * * 1 /sbin/reboot"
|
|
local CRON_DEPLOY="@reboot sleep 60 && cd /opt/PaperClip && git fetch origin && git reset --hard origin/main && /bin/bash /opt/PaperClip/deploy_script.sh >> /var/log/paperclip-deploy.log 2>&1"
|
|
|
|
# Remove old copies
|
|
crontab -l 2>/dev/null | grep -v -E "(/opt/PaperClip/deploy_script.sh|/sbin/reboot)" | crontab - || true
|
|
|
|
# Install new cron jobs
|
|
(crontab -l 2>/dev/null; echo "$CRON_REBOOT"; echo "$CRON_DEPLOY") | crontab -
|
|
}
|
|
|
|
###############################################################################
|
|
# 3. Handle AGE private key
|
|
###############################################################################
|
|
handle_age_key() {
|
|
log "Checking AGE private key..."
|
|
mkdir -p "$(dirname "${AGE_PRIVATE_KEY}")"
|
|
|
|
if [[ ! -f "${AGE_PRIVATE_KEY}" ]]; then
|
|
warn "AGE private key not found."
|
|
echo -n "Please paste your AGE private key now (input hidden): "
|
|
read -rs age_key_input
|
|
echo ""
|
|
if [[ -z "${age_key_input}" ]]; then
|
|
die "No key provided. Aborting."
|
|
fi
|
|
echo "${age_key_input}" > "${AGE_PRIVATE_KEY}"
|
|
chmod 600 "${AGE_PRIVATE_KEY}"
|
|
fi
|
|
|
|
if ! head -1 "${AGE_PRIVATE_KEY}" | grep -q "^AGE-SECRET-KEY-"; then
|
|
die "Invalid AGE private key format. Must start with AGE-SECRET-KEY-"
|
|
fi
|
|
chmod 600 "${AGE_PRIVATE_KEY}"
|
|
}
|
|
|
|
###############################################################################
|
|
# 4. Git clone / pull
|
|
###############################################################################
|
|
update_repo() {
|
|
log "Updating repository..."
|
|
local reexec_needed=false
|
|
|
|
if [[ ! -d "${REPO_DIR}/.git" ]]; then
|
|
log "No .git directory found — cloning repository..."
|
|
local tmp_dir
|
|
tmp_dir="$(mktemp -d)"
|
|
git clone "${GIT_REPO_URL}" "${tmp_dir}/repo"
|
|
rsync -a --ignore-existing "${tmp_dir}/repo/" "${REPO_DIR}/"
|
|
cp -a "${tmp_dir}/repo/.git" "${REPO_DIR}/.git"
|
|
rm -rf "${tmp_dir}"
|
|
cd "${REPO_DIR}"
|
|
git checkout -- . 2>/dev/null || true
|
|
reexec_needed=true
|
|
else
|
|
cd "${REPO_DIR}"
|
|
local old_commit
|
|
old_commit=$(git rev-parse HEAD 2>/dev/null || echo "old")
|
|
git fetch origin
|
|
git reset --hard origin/main || git reset --hard origin/master || true
|
|
local new_commit
|
|
new_commit=$(git rev-parse HEAD 2>/dev/null || echo "new")
|
|
|
|
if [[ "${old_commit}" != "${new_commit}" ]]; then
|
|
log "Update detected! (${old_commit:0:7} -> ${new_commit:0:7})"
|
|
reexec_needed=true
|
|
fi
|
|
fi
|
|
chmod 700 "${REPO_DIR}/deploy_script.sh" 2>/dev/null || true
|
|
|
|
if [[ "${reexec_needed}" == "true" ]]; then
|
|
if [[ -z "${PAPERCLIP_REEXEC:-}" ]]; then
|
|
log "Re-executing updated script..."
|
|
export PAPERCLIP_REEXEC=1
|
|
exec bash "$0" "$@"
|
|
else
|
|
warn "Re-exec already attempted. Continuing to avoid infinite loop."
|
|
fi
|
|
fi
|
|
}
|
|
|
|
###############################################################################
|
|
# 5. Docker network & Discover services
|
|
###############################################################################
|
|
ensure_network() {
|
|
if ! docker network inspect "${DOCKER_NETWORK}" &>/dev/null; then
|
|
log "Creating Docker network: ${DOCKER_NETWORK}"
|
|
docker network create "${DOCKER_NETWORK}"
|
|
fi
|
|
}
|
|
|
|
discover_services() {
|
|
ACTIVE_SERVICES=()
|
|
for svc_dir in "${SERVICES_DIR}"/*/; do
|
|
svc_name="$(basename "${svc_dir}")"
|
|
if is_active_service "${svc_dir}"; then
|
|
ACTIVE_SERVICES+=("${svc_name}")
|
|
fi
|
|
done
|
|
log "Active services: ${ACTIVE_SERVICES[*]}"
|
|
}
|
|
|
|
cleanup_deactivated() {
|
|
log "Checking for deactivated services..."
|
|
local running_containers
|
|
running_containers=$(docker ps --format '{{.Names}}' 2>/dev/null || true)
|
|
for svc_dir in "${SERVICES_DIR}"/*/; do
|
|
svc_name="$(basename "${svc_dir}")"
|
|
if ! is_active_service "${svc_dir}" && [[ -f "${svc_dir}/docker-compose.yml" ]]; then
|
|
if echo "${running_containers}" | grep -q "${svc_name}"; then
|
|
log "Stopping deactivated service: ${svc_name}"
|
|
cd "${svc_dir}"
|
|
docker compose down --remove-orphans 2>/dev/null || true
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
###############################################################################
|
|
# 6. Maintenance Mode
|
|
###############################################################################
|
|
enable_maintenance_mode() {
|
|
log "Enabling maintenance mode on all active subdomains..."
|
|
local all_domains
|
|
all_domains=$(grep -hEo "([a-zA-Z0-9.-]+\.orfel\.de|orfel\.de)" "${SERVICES_DIR}"/*/*.caddy 2>/dev/null | sort -u | tr '\n' ' ' | sed 's/ $//')
|
|
if [[ -z "${all_domains}" ]]; then return; fi
|
|
|
|
mkdir -p "${CADDY_FRAGMENTS_DIR}"
|
|
find "${CADDY_FRAGMENTS_DIR}" -name '*.caddy' -type f -delete 2>/dev/null || true
|
|
|
|
cat << 'EOF' > "${CADDY_FRAGMENTS_DIR}/maintenance.caddy"
|
|
DOMAINS_PLACEHOLDER {
|
|
header Content-Type text/html
|
|
respond "<!DOCTYPE html><html><head><meta charset=\"utf-8\"><title>Wartungsarbeiten</title><style>body{background:#1a202c;color:#e2e8f0;font-family:system-ui,-apple-system,sans-serif;display:flex;align-items:center;justify-content:center;height:100vh;margin:0}.box{text-align:center;padding:2rem;background:#2d3748;border-radius:1rem;box-shadow:0 10px 15px -3px rgba(0,0,0,0.1)}h1{margin-top:0;color:#63b3ed}</style></head><body><div class=\"box\"><h1>Wartungsarbeiten</h1><p>Das System wird gerade aktualisiert.<br>Die Seite ist in wenigen Minuten wieder erreichbar.</p></div></body></html>" 503
|
|
}
|
|
EOF
|
|
sed -i "s|DOMAINS_PLACEHOLDER|${all_domains}|g" "${CADDY_FRAGMENTS_DIR}/maintenance.caddy"
|
|
|
|
if docker ps --format '{{.Names}}' | grep -q "^caddy$"; then
|
|
if ! docker exec caddy caddy reload --config /etc/caddy/Caddyfile 2>/dev/null; then
|
|
log "Caddy reload failed (maybe admin API is disabled). Restarting Caddy..."
|
|
docker restart caddy
|
|
fi
|
|
elif [[ " ${ACTIVE_SERVICES[*]} " =~ " caddy " ]]; then
|
|
cd "${SERVICES_DIR}/caddy"
|
|
docker compose up -d --remove-orphans 2>/dev/null || true
|
|
fi
|
|
sleep 1
|
|
}
|
|
|
|
stop_all_services() {
|
|
log "Stopping active containers (excluding infra)..."
|
|
for svc_name in "${ACTIVE_SERVICES[@]}"; do
|
|
if [[ " ${INFRA_SERVICES[*]} " =~ " ${svc_name} " ]]; then continue; fi
|
|
cd "${SERVICES_DIR}/${svc_name}"
|
|
docker compose down --remove-orphans 2>/dev/null || true
|
|
done
|
|
}
|
|
|
|
###############################################################################
|
|
# 7. Secrets & Data
|
|
###############################################################################
|
|
decrypt_envs() {
|
|
# -------------------------------------------------------------------------
|
|
# 1. Geteilte API Keys entschlüsseln (Single Source of Truth)
|
|
# -------------------------------------------------------------------------
|
|
local shared_env="${SERVICES_DIR}/shared.env"
|
|
local shared_env_age="${SERVICES_DIR}/shared.env.age"
|
|
if [[ -f "${shared_env_age}" ]]; then
|
|
log "Decrypting shared API keys (shared.env.age)..."
|
|
age --decrypt -i "${AGE_PRIVATE_KEY}" -o "${shared_env}" "${shared_env_age}"
|
|
sed -i 's/\r$//' "${shared_env}"
|
|
chmod 600 "${shared_env}"
|
|
else
|
|
warn "shared.env.age not found — no shared API keys loaded."
|
|
fi
|
|
|
|
# -------------------------------------------------------------------------
|
|
# 2. Service-spezifische .env.age Dateien entschlüsseln
|
|
# -------------------------------------------------------------------------
|
|
log "Decrypting service-specific .env.age files..."
|
|
for svc_name in "${ACTIVE_SERVICES[@]}"; do
|
|
local svc_dir="${SERVICES_DIR}/${svc_name}"
|
|
if [[ -f "${svc_dir}/.env.age" ]]; then
|
|
age --decrypt -i "${AGE_PRIVATE_KEY}" -o "${svc_dir}/.env" "${svc_dir}/.env.age"
|
|
sed -i 's/\r$//' "${svc_dir}/.env"
|
|
chmod 600 "${svc_dir}/.env"
|
|
fi
|
|
done
|
|
}
|
|
|
|
create_volumes() {
|
|
log "Creating persistent volume directories..."
|
|
for svc_name in "${ACTIVE_SERVICES[@]}"; do
|
|
mkdir -p "${DATA_BASE}/${svc_name}"
|
|
done
|
|
}
|
|
|
|
###############################################################################
|
|
# 8. Start Infra & Deploy
|
|
###############################################################################
|
|
pull_images() {
|
|
log "Pulling Docker images..."
|
|
for svc_name in "${ACTIVE_SERVICES[@]}"; do
|
|
cd "${SERVICES_DIR}/${svc_name}"
|
|
if [[ -f "Dockerfile" ]]; then
|
|
docker compose build --no-cache || SKIPPED_SERVICES+=("${svc_name}")
|
|
else
|
|
docker compose pull || true
|
|
fi
|
|
done
|
|
}
|
|
|
|
start_infra() {
|
|
# Postgres is started via depends_on in paperclip service
|
|
true
|
|
}
|
|
|
|
assemble_caddy_fragments() {
|
|
log "Assembling Caddy fragments..."
|
|
mkdir -p "${CADDY_FRAGMENTS_DIR}"
|
|
rm -f "${CADDY_FRAGMENTS_DIR}"/*.caddy
|
|
for svc_name in "${ACTIVE_SERVICES[@]}"; do
|
|
if [[ -f "${SERVICES_DIR}/${svc_name}/${svc_name}.caddy" ]]; then
|
|
cp "${SERVICES_DIR}/${svc_name}/${svc_name}.caddy" "${CADDY_FRAGMENTS_DIR}/"
|
|
fi
|
|
done
|
|
}
|
|
|
|
disable_maintenance_mode() {
|
|
log "Switching Caddy to production..."
|
|
assemble_caddy_fragments
|
|
if [[ " ${ACTIVE_SERVICES[*]} " =~ " caddy " ]]; then
|
|
cd "${SERVICES_DIR}/caddy"
|
|
docker compose up -d --remove-orphans
|
|
sleep 2
|
|
if ! docker exec caddy caddy reload --config /etc/caddy/Caddyfile 2>/dev/null; then
|
|
log "Caddy reload failed. Restarting Caddy..."
|
|
docker restart caddy
|
|
fi
|
|
fi
|
|
}
|
|
|
|
start_remaining() {
|
|
log "Starting remaining services..."
|
|
for svc_name in "${ACTIVE_SERVICES[@]}"; do
|
|
if [[ " ${INFRA_SERVICES[*]} " =~ " ${svc_name} " ]]; then continue; fi
|
|
cd "${SERVICES_DIR}/${svc_name}"
|
|
if ! docker compose up -d --remove-orphans; then
|
|
SKIPPED_SERVICES+=("${svc_name} (Start failed)")
|
|
fi
|
|
done
|
|
}
|
|
|
|
###############################################################################
|
|
# 9. Service-Init-Skripte ausführen (Pre & Post)
|
|
###############################################################################
|
|
run_service_pre_starts() {
|
|
log "Running service.pre_start scripts..."
|
|
for svc_name in "${ACTIVE_SERVICES[@]}"; do
|
|
local pre_script="${SERVICES_DIR}/${svc_name}/service.pre_start"
|
|
if [[ -f "${pre_script}" ]]; then
|
|
log "--- Running pre_start for: ${svc_name} ---"
|
|
chmod +x "${pre_script}"
|
|
bash "${pre_script}" || warn "service.pre_start for '${svc_name}' exited with errors (non-fatal)."
|
|
fi
|
|
done
|
|
}
|
|
|
|
run_service_inits() {
|
|
log "Running service.init scripts..."
|
|
for svc_name in "${ACTIVE_SERVICES[@]}"; do
|
|
local init_script="${SERVICES_DIR}/${svc_name}/service.init"
|
|
if [[ -f "${init_script}" ]]; then
|
|
log "--- Running init for: ${svc_name} ---"
|
|
chmod +x "${init_script}"
|
|
bash "${init_script}" || warn "service.init for '${svc_name}' exited with errors (non-fatal)."
|
|
fi
|
|
done
|
|
}
|
|
|
|
###############################################################################
|
|
# MAIN
|
|
###############################################################################
|
|
main() {
|
|
log "=========================================="
|
|
log " PaperClip Deploy Script"
|
|
log "=========================================="
|
|
if [[ "$(id -u)" -ne 0 ]]; then die "Run as root (sudo)."; fi
|
|
|
|
trap on_deploy_failure ERR
|
|
|
|
if [[ -z "${PAPERCLIP_REEXEC:-}" ]]; then
|
|
notify "PaperClip Deploy Start" "Deploy-Skript gestartet. System wird überprüft..." "default" "gear"
|
|
else
|
|
notify "PaperClip Deploy Neustart" "Skript hat sich selbst geupdatet und führt sich nun aus." "default" "arrows_counterclockwise"
|
|
fi
|
|
|
|
install_packages
|
|
configure_system
|
|
handle_age_key
|
|
update_repo
|
|
ensure_network
|
|
discover_services
|
|
cleanup_deactivated
|
|
enable_maintenance_mode
|
|
stop_all_services
|
|
decrypt_envs
|
|
run_service_pre_starts
|
|
create_volumes
|
|
pull_images
|
|
start_infra
|
|
start_remaining
|
|
disable_maintenance_mode
|
|
docker image prune -af 2>/dev/null || true
|
|
run_service_inits
|
|
|
|
trap - ERR
|
|
log "Deployment complete! Active services: ${ACTIVE_SERVICES[*]}"
|
|
notify "PaperClip Deploy ERFOLGREICH" "Deployment abgeschlossen! Services: ${ACTIVE_SERVICES[*]}" "default" "white_check_mark"
|
|
}
|
|
|
|
main "$@"
|