setup.sh

#!/bin/bash

# N8N with Cloudflare Tunnel Setup Script
# This script sets up the complete directory structure and files for N8N

set -e

echo "🚀 Setting up N8N with Cloudflare Tunnel..."

# Create directory structure
echo "📁 Creating directory structure..."
mkdir -p data/{postgres,n8n,redis,traefik,prometheus,grafana}
mkdir -p n8n-custom
mkdir -p n8n-nodes
mkdir -p n8n-backups
mkdir -p local-files
mkdir -p traefik/dynamic
mkdir -p cloudflared
mkdir -p prometheus
mkdir -p grafana/{provisioning/dashboards,provisioning/datasources}
mkdir -p logs
mkdir -p backups

# Set proper permissions
echo "🔒 Setting permissions..."
chmod 755 data/{postgres,n8n,redis,traefik,prometheus,grafana}
chmod 755 n8n-custom n8n-nodes n8n-backups local-files
chmod 755 traefik/dynamic
chmod 755 cloudflared
chmod 755 logs backups

# Create traefik acme.json file
touch data/traefik/acme.json
chmod 600 data/traefik/acme.json

# Create traefik dynamic configuration
echo "🚀 Creating Traefik dynamic configuration..."
cat > traefik/dynamic/tls.yml << 'EOF'
tls:
  options:
    default:
      sslStrategies:
        - "tls.SniStrict"
      minVersion: "VersionTLS12"
      cipherSuites:
        - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
        - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
        - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"
        - "TLS_RSA_WITH_AES_256_GCM_SHA384"
        - "TLS_RSA_WITH_AES_128_GCM_SHA256"
      curvePreferences:
        - "CurveP521"
        - "CurveP384"
EOF

# Create prometheus configuration
echo "📊 Creating Prometheus configuration..."
cat > prometheus/prometheus.yml << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'traefik'
    static_configs:
      - targets: ['traefik:8080']
    metrics_path: '/metrics'

  - job_name: 'n8n'
    static_configs:
      - targets: ['n8n:5678']
    metrics_path: '/metrics'

  - job_name: 'postgres'
    static_configs:
      - targets: ['postgres:5432']

  - job_name: 'redis'
    static_configs:
      - targets: ['redis:6379']
EOF

# Create grafana datasource
echo "📈 Creating Grafana datasource..."
cat > grafana/provisioning/datasources/prometheus.yml << 'EOF'
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    jsonData:
      timeInterval: "5s"
EOF

# Create grafana dashboard provisioning
cat > grafana/provisioning/dashboards/dashboard.yml << 'EOF'
apiVersion: 1
providers:
  - name: 'default'
    orgId: 1
    folder: ''
    type: file
    disableDeletion: false
    updateIntervalSeconds: 10
    options:
      path: /etc/grafana/provisioning/dashboards
EOF

# Create postgres init script
echo "🗄️ Creating postgres init script..."
cat > init-data.sh << 'EOF'
#!/bin/bash
set -e

# Create database and user if they don't exist
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
    CREATE USER $POSTGRES_NON_ROOT_USER WITH PASSWORD '$POSTGRES_NON_ROOT_PASSWORD';
    GRANT ALL PRIVILEGES ON DATABASE $POSTGRES_DB TO $POSTGRES_NON_ROOT_USER;
    ALTER USER $POSTGRES_NON_ROOT_USER CREATEDB;
EOSQL
EOF

chmod +x init-data.sh

# Create Cloudflare tunnel config template
echo "☁️ Creating Cloudflare tunnel configuration template..."
cat > cloudflared/config.yml << 'EOF'
# Cloudflare Tunnel Configuration
tunnel: ${CLOUDFLARE_TUNNEL_ID}
credentials-file: /etc/cloudflared/credentials.json

ingress:
  # N8N Application
  - hostname: ${SUBDOMAIN}.${DOMAIN_NAME}
    service: http://n8n:5678
    originRequest:
      connectTimeout: 30s
      tlsTimeout: 10s
      tcpKeepAlive: 30s
      keepAliveTimeout: 90s
      keepAliveConnections: 100
      httpHostHeader: ${SUBDOMAIN}.${DOMAIN_NAME}
  
  # Traefik Dashboard (optional)
  - hostname: traefik.${DOMAIN_NAME}
    service: http://traefik:8080
    originRequest:
      httpHostHeader: traefik.${DOMAIN_NAME}
  
  # Grafana Dashboard (optional)
  - hostname: grafana.${DOMAIN_NAME}
    service: http://grafana:3000
    originRequest:
      httpHostHeader: grafana.${DOMAIN_NAME}
  
  # Prometheus (optional)
  - hostname: prometheus.${DOMAIN_NAME}
    service: http://prometheus:9090
    originRequest:
      httpHostHeader: prometheus.${DOMAIN_NAME}
  
  # Catch-all rule (must be last)
  - service: http_status:404

# Optional: Configure tunnel-wide settings
logLevel: info
retries: 3
EOF

# Create credentials file template
cat > cloudflared/credentials.json << 'EOF'
{
  "AccountTag": "your-account-tag",
  "TunnelSecret": "your-tunnel-secret",
  "TunnelID": "your-tunnel-id"
}
EOF

chmod 600 cloudflared/credentials.json

# Create maintenance scripts
echo "🔧 Creating maintenance scripts..."

# Create deployment scripts
echo "🚀 Creating deployment scripts..."
cat > deploy.sh << 'EOF'
#!/bin/bash
# Main deployment script

set -e

echo "🚀 Starting N8N deployment..."

# Check if .env file exists
if [ ! -f .env ]; then
    echo "❌ .env file not found! Please copy .env.example to .env and configure it."
    exit 1
fi

# Load environment variables
source .env

# Validate required variables
if [ -z "$DOMAIN_NAME" ] || [ -z "$SUBDOMAIN" ] || [ -z "$SSL_EMAIL" ]; then
    echo "❌ Required environment variables not set. Please check your .env file."
    exit 1
fi

# Create traefik acme.json if it doesn't exist
touch data/traefik/acme.json
chmod 600 data/traefik/acme.json

# Pull latest images
echo "🔄 Pulling latest images..."
docker-compose pull

# Start services
echo "🔧 Starting services..."
docker-compose up -d

# Wait for services to be ready
echo "⏳ Waiting for services to be ready..."
sleep 30

# Check service health
echo "🔍 Checking service health..."
docker-compose ps

# Show logs for debugging
echo "📋 Recent logs:"
docker-compose logs --tail=50

echo "✅ Deployment completed!"
echo ""
echo "🌐 Access your services:"
echo "- N8N: https://${SUBDOMAIN}.${DOMAIN_NAME}"
echo "- Traefik Dashboard: https://traefik.${DOMAIN_NAME}"
echo ""
echo "📋 To enable optional services:"
echo "- Monitoring: docker-compose --profile monitoring up -d"
echo "- Cloudflare Tunnel: docker-compose --profile cloudflare up -d"
echo "- Auto-updates: docker-compose --profile watchtower up -d"
echo ""
echo "🔍 Check status: docker-compose ps"
echo "📋 View logs: docker-compose logs -f [service_name]"
EOF

cat > deploy-monitoring.sh << 'EOF'
#!/bin/bash
# Deploy with monitoring stack

set -e

echo "📊 Starting N8N with monitoring..."

# Check if .env file exists
if [ ! -f .env ]; then
    echo "❌ .env file not found! Please copy .env.example to .env and configure it."
    exit 1
fi

# Load environment variables
source .env

# Start main services
echo "🔧 Starting main services..."
docker-compose up -d

# Start monitoring services
echo "📊 Starting monitoring services..."
docker-compose --profile monitoring up -d

# Wait for services to be ready
echo "⏳ Waiting for services to be ready..."
sleep 45

# Check service health
echo "🔍 Checking service health..."
docker-compose ps

echo "✅ Monitoring deployment completed!"
echo ""
echo "🌐 Access your services:"
echo "- N8N: https://${SUBDOMAIN}.${DOMAIN_NAME}"
echo "- Traefik Dashboard: https://traefik.${DOMAIN_NAME}"
echo "- Grafana: https://grafana.${DOMAIN_NAME}"
echo "- Prometheus: https://prometheus.${DOMAIN_NAME}"
echo ""
echo "📊 Default Grafana login: admin / ${GRAFANA_PASSWORD:-admin}"
EOF

cat > deploy-cloudflare.sh << 'EOF'
#!/bin/bash
# Deploy with Cloudflare tunnel

set -e

echo "☁️ Starting N8N with Cloudflare tunnel..."

# Check if .env file exists
if [ ! -f .env ]; then
    echo "❌ .env file not found! Please copy .env.example to .env and configure it."
    exit 1
fi

# Check if Cloudflare tunnel is configured
if [ ! -f cloudflared/credentials.json ]; then
    echo "❌ Cloudflare tunnel not configured! Please set up your tunnel credentials."
    exit 1
fi

# Load environment variables
source .env

# Update cloudflared config with environment variables
envsubst < cloudflared/config.yml > cloudflared/config.yml.tmp
mv cloudflared/config.yml.tmp cloudflared/config.yml

# Start main services (without traefik ports for tunnel mode)
echo "🔧 Starting main services..."
docker-compose up -d postgres redis n8n

# Start cloudflare tunnel
echo "☁️ Starting Cloudflare tunnel..."
docker-compose --profile cloudflare up -d

# Wait for services to be ready
echo "⏳ Waiting for services to be ready..."
sleep 30

# Check service health
echo "🔍 Checking service health..."
docker-compose ps

echo "✅ Cloudflare tunnel deployment completed!"
echo ""
echo "🌐 Access your N8N via your configured Cloudflare domain:"
echo "- N8N: https://${SUBDOMAIN}.${DOMAIN_NAME}"
echo ""
echo "📋 Cloudflare tunnel logs: docker-compose logs -f cloudflared"
EOF

chmod +x deploy.sh deploy-monitoring.sh deploy-cloudflare.sh

cat > backup.sh << 'EOF'
#!/bin/bash
# Backup script for N8N data

BACKUP_DIR="./backups/$(date +%Y%m%d_%H%M%S)"
mkdir -p "$BACKUP_DIR"

echo "📦 Creating backup in $BACKUP_DIR..."

# Backup database
echo "🗄️ Backing up database..."
docker exec n8n_postgres pg_dump -U n8n n8n | gzip > "$BACKUP_DIR/database.sql.gz"

# Backup n8n data
echo "📁 Backing up n8n data..."
tar -czf "$BACKUP_DIR/n8n_data.tar.gz" -C data/n8n . 2>/dev/null || echo "⚠️ n8n data directory empty or not found"

# Backup traefik data
echo "🔒 Backing up traefik data..."
tar -czf "$BACKUP_DIR/traefik_data.tar.gz" -C data/traefik . 2>/dev/null || echo "⚠️ traefik data directory empty or not found"

# Backup configurations
echo "⚙️ Backing up configurations..."
cp -r cloudflared "$BACKUP_DIR/" 2>/dev/null || echo "⚠️ cloudflared config not found"
cp -r traefik "$BACKUP_DIR/" 2>/dev/null || echo "⚠️ traefik config not found"
cp -r prometheus "$BACKUP_DIR/" 2>/dev/null || echo "⚠️ prometheus config not found"
cp -r grafana "$BACKUP_DIR/" 2>/dev/null || echo "⚠️ grafana config not found"
cp .env "$BACKUP_DIR/" 2>/dev/null || echo "⚠️ .env file not found"

# Create backup info file
cat > "$BACKUP_DIR/backup_info.txt" << EOF
Backup created: $(date)
N8N Version: $(docker inspect n8n_app | grep -i version | head -1 || echo "Unknown")
Database: PostgreSQL
Services backed up: postgres, n8n, traefik, configurations
EOF

echo "✅ Backup completed successfully in $BACKUP_DIR!"
echo "📊 Backup size: $(du -sh "$BACKUP_DIR" | cut -f1)"
EOF

cat > restore.sh << 'EOF'
#!/bin/bash
# Restore script for N8N data

if [ $# -eq 0 ]; then
    echo "Usage: $0 <backup_directory>"
    echo "Available backups:"
    ls -la backups/ 2>/dev/null || echo "No backups found"
    exit 1
fi

BACKUP_DIR="$1"

if [ ! -d "$BACKUP_DIR" ]; then
    echo "❌ Backup directory $BACKUP_DIR does not exist"
    exit 1
fi

echo "📦 Restoring from $BACKUP_DIR..."

# Confirm restore
read -p "⚠️ This will overwrite existing data. Continue? (y/N): " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
    echo "Restore cancelled."
    exit 1
fi

# Stop services
echo "🛑 Stopping services..."
docker-compose down

# Wait for services to stop
sleep 10

# Restore database
if [ -f "$BACKUP_DIR/database.sql.gz" ]; then
    echo "🗄️ Restoring database..."
    docker-compose up -d postgres
    sleep 15
    
    # Drop and recreate database
    docker exec n8n_postgres psql -U n8n -d postgres -c "DROP DATABASE IF EXISTS n8n;"
    docker exec n8n_postgres psql -U n8n -d postgres -c "CREATE DATABASE n8n;"
    
    # Restore database
    gunzip -c "$BACKUP_DIR/database.sql.gz" | docker exec -i n8n_postgres psql -U n8n -d n8n
    
    docker-compose stop postgres
else
    echo "⚠️ Database backup not found, skipping database restore"
fi

# Restore n8n data
if [ -f "$BACKUP_DIR/n8n_data.tar.gz" ]; then
    echo "📁 Restoring n8n data..."
    rm -rf data/n8n/*
    tar -xzf "$BACKUP_DIR/n8n_data.tar.gz" -C data/n8n/
else
    echo "⚠️ N8N data backup not found, skipping n8n data restore"
fi

# Restore traefik data
if [ -f "$BACKUP_DIR/traefik_data.tar.gz" ]; then
    echo "🔒 Restoring traefik data..."
    rm -rf data/traefik/*
    tar -xzf "$BACKUP_DIR/traefik_data.tar.gz" -C data/traefik/
    chmod 600 data/traefik/acme.json 2>/dev/null || true
else
    echo "⚠️ Traefik data backup not found, skipping traefik data restore"
fi

# Restore configurations
echo "⚙️ Restoring configurations..."
cp -r "$BACKUP_DIR/cloudflared" . 2>/dev/null || echo "⚠️ cloudflared config not found in backup"
cp -r "$BACKUP_DIR/traefik" . 2>/dev/null || echo "⚠️ traefik config not found in backup"
cp -r "$BACKUP_DIR/prometheus" . 2>/dev/null || echo "⚠️ prometheus config not found in backup"
cp -r "$BACKUP_DIR/grafana" . 2>/dev/null || echo "⚠️ grafana config not found in backup"
cp "$BACKUP_DIR/.env" . 2>/dev/null || echo "⚠️ .env file not found in backup"

# Set proper permissions
chmod 600 cloudflared/credentials.json 2>/dev/null || true
chmod 600 data/traefik/acme.json 2>/dev/null || true

echo "✅ Restore completed successfully!"
echo "🚀 Start services with: docker-compose up -d"
EOF

chmod +x backup.sh restore.sh

# Create docker-compose override for development
cat > docker-compose.override.yml << 'EOF'
version: '3.8'

services:
  n8n:
    environment:
      - N8N_LOG_LEVEL=debug
      - NODE_ENV=development
    volumes:
      - ./logs:/var/log/n8n
  
  postgres:
    ports:
      - "5432:5432"
    
  redis:
    ports:
      - "6379:6379"
      
  traefik:
    ports:
      - "8080:8080"  # Traefik dashboard
EOF

# Create systemd service file
cat > n8n.service << 'EOF'
[Unit]
Description=N8N Workflow Automation
Requires=docker.service
After=docker.service

[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=/path/to/your/n8n/directory
ExecStart=/usr/local/bin/docker-compose up -d
ExecStop=/usr/local/bin/docker-compose down
TimeoutStartSec=0
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

# Create update script
cat > update.sh << 'EOF'
#!/bin/bash
# Update script for N8N and services

set -e

echo "🔄 Updating N8N and services..."

# Create backup before update
echo "📦 Creating backup before update..."
./backup.sh

# Pull latest images
echo "⬇️ Pulling latest images..."
docker-compose pull

# Recreate containers with new images
echo "🔄 Recreating containers..."
docker-compose up -d --force-recreate

# Wait for services to be ready
echo "⏳ Waiting for services to be ready..."
sleep 30

# Check service health
echo "🔍 Checking service health..."
docker-compose ps

# Clean up old images
echo "🧹 Cleaning up old images..."
docker image prune -f

echo "✅ Update completed successfully!"
EOF

chmod +x update.sh

# Create health check script
cat > health-check.sh << 'EOF'
#!/bin/bash
# Health check script for N8N services

echo "🔍 Checking N8N services health..."

# Check if services are running
services=("n8n_postgres" "n8n_redis" "n8n_app" "n8n_traefik")
for service in "${services[@]}"; do
    if docker ps --filter "name=$service" --filter "status=running" | grep -q "$service"; then
        echo "✅ $service is running"
    else
        echo "❌ $service is not running"
    fi
done

# Check service health
echo ""
echo "📊 Service health status:"
docker-compose ps

# Check logs for errors
echo ""
echo "📋 Recent error logs:"
docker-compose logs --tail=10 --since=5m | grep -i error || echo "No recent errors found"

# Check disk usage
echo ""
echo "💾 Disk usage:"
echo "Docker volumes:"
docker system df -v | grep -E "(VOLUME NAME|n8n|traefik|postgres|redis|grafana|prometheus)"

echo ""
echo "Local directories:"
du -sh data/ logs/ backups/ 2>/dev/null || echo "Directories not found"

# Check network connectivity
echo ""
echo "🌐 Network connectivity:"
if docker exec n8n_app curl -s http://localhost:5678/healthz > /dev/null; then
    echo "✅ N8N health endpoint responding"
else
    echo "❌ N8N health endpoint not responding"
fi

# Check SSL certificates
echo ""
echo "🔒 SSL certificate status:"
if [ -f "data/traefik/acme.json" ]; then
    cert_count=$(jq -r '.mytlschallenge.Certificates | length' data/traefik/acme.json 2>/dev/null || echo "0")
    echo "SSL certificates: $cert_count"
else
    echo "SSL certificate file not found"
fi
EOF

chmod +x health-check.sh

# Create log management script
cat > manage-logs.sh << 'EOF'
#!/bin/bash
# Log management script

case "$1" in
    "tail")
        echo "📋 Tailing logs for all services..."
        docker-compose logs -f
        ;;
    "n8n")
        echo "📋 N8N logs:"
        docker-compose logs -f n8n
        ;;
    "traefik")
        echo "📋 Traefik logs:"
        docker-compose logs -f traefik
        ;;
    "postgres")
        echo "📋 PostgreSQL logs:"
        docker-compose logs -f postgres
        ;;
    "redis")
        echo "📋 Redis logs:"
        docker-compose logs -f redis
        ;;
    "cloudflared")
        echo "📋 Cloudflare tunnel logs:"
        docker-compose logs -f cloudflared
        ;;
    "clean")
        echo "🧹 Cleaning old logs..."
        docker-compose logs --no-color > logs/all_services_$(date +%Y%m%d).log
        docker system prune -f
        echo "Logs archived to logs/all_services_$(date +%Y%m%d).log"
        ;;
    *)
        echo "Usage: $0 {tail|n8n|traefik|postgres|redis|cloudflared|clean}"
        echo ""
        echo "Commands:"
        echo "  tail      - Show logs for all services"
        echo "  n8n       - Show N8N logs"
        echo "  traefik   - Show Traefik logs"
        echo "  postgres  - Show PostgreSQL logs"
        echo "  redis     - Show Redis logs"
        echo "  cloudflared - Show Cloudflare tunnel logs"
        echo "  clean     - Archive logs and clean system"
        ;;
esac
EOF

chmod +x manage-logs.sh

echo "📋 Setup completed! Next steps:"
echo ""
echo "1. Configure your environment:"
echo "   - Copy .env.example to .env"
echo "   - Set your domain name and email"
echo "   - Generate strong passwords"
echo "   - Generate N8N encryption key: openssl rand -base64 32"
echo "   - Create Traefik basic auth hash: htpasswd -nb admin password"
echo ""
echo "2. Set up Cloudflare tunnel (optional):"
echo "   - Go to Cloudflare Zero Trust Dashboard"
echo "   - Create a new tunnel"
echo "   - Download credentials.json to cloudflared/ directory"
echo "   - Update cloudflared/config.yml with your tunnel ID"
echo ""
echo "3. Choose your deployment method:"
echo "   - Standard with Traefik: ./deploy.sh"
echo "   - With monitoring: ./deploy-monitoring.sh"
echo "   - With Cloudflare tunnel: ./deploy-cloudflare.sh"
echo ""
echo "4. Access your services:"
echo "   - N8N: https://${SUBDOMAIN:-n8n}.${DOMAIN_NAME:-yourdomain.com}"
echo "   - Traefik Dashboard: https://traefik.${DOMAIN_NAME:-yourdomain.com}"
echo "   - Grafana (if enabled): https://grafana.${DOMAIN_NAME:-yourdomain.com}"
echo ""
echo "📁 Directory structure created:"
echo "├── data/              (persistent data)"
echo "├── local-files/       (n8n file storage)"
echo "├── n8n-custom/        (custom n8n files)"
echo "├── n8n-nodes/         (custom nodes)"
echo "├── n8n-backups/       (n8n backups)"
echo "├── traefik/           (traefik configuration)"
echo "├── prometheus/        (prometheus config)"
echo "├── grafana/           (grafana config)"
echo "├── cloudflared/       (tunnel configuration)"
echo "├── logs/              (application logs)"
echo "└── backups/           (backup storage)"
echo ""
echo "🔧 Scripts created:"
echo "├── deploy.sh          (main deployment)"
echo "├── deploy-monitoring.sh (with monitoring)"
echo "├── deploy-cloudflare.sh (cloudflare tunnel)"
echo "├── backup.sh          (backup your data)"
echo "├── restore.sh         (restore from backup)"
echo "├── update.sh          (update services)"
echo "├── health-check.sh    (check service health)"
echo "├── manage-logs.sh     (log management)"
echo "└── n8n.service        (systemd service file)"
echo ""
echo "⚠️  Important security notes:"
echo "- Change all default passwords in .env file"
echo "- Use strong encryption keys"
echo "- Enable firewall rules"
echo "- Regularly backup your data"
echo "- Monitor service logs"
echo ""
echo "🚀 Quick start:"
echo "1. cp .env.example .env"
echo "2. nano .env  # Configure your settings"
echo "3. ./deploy.sh"
echo ""
echo "📚 Useful commands:"
echo "- Check status: docker-compose ps"
echo "- View logs: ./manage-logs.sh tail"
echo "- Backup data: ./backup.sh"
echo "- Health check: ./health-check.sh"
echo "- Update services: ./update.sh"