From b5c1ac95b7c96fab75dcd7a7197cf59758230345 Mon Sep 17 00:00:00 2001 From: Julien Gautier Date: Tue, 28 Apr 2026 21:24:57 +0200 Subject: [PATCH] dbdump initial commit --- dbdump_mongo/.env.example | 18 ++ dbdump_mongo/.gitignore | 15 ++ dbdump_mongo/README.md | 219 +++++++++++++++++++ dbdump_mongo/db-dump.sh | 450 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 702 insertions(+) create mode 100644 dbdump_mongo/.env.example create mode 100644 dbdump_mongo/.gitignore create mode 100644 dbdump_mongo/README.md create mode 100755 dbdump_mongo/db-dump.sh diff --git a/dbdump_mongo/.env.example b/dbdump_mongo/.env.example new file mode 100644 index 0000000..eeeb808 --- /dev/null +++ b/dbdump_mongo/.env.example @@ -0,0 +1,18 @@ +# MongoDB Dump Environment Configuration Example +# Copy this pattern to create environments: .env.prod, .env.uat, etc. +# DO NOT commit actual configuration files to version control + +# Environment: example +# Created: 2026-04-14 + +# MongoDB credentials (no defaults - must be provided) +MONGO_USERNAME=admin_user +MONGO_PASSWORD=secure_password_here +MONGO_HOST=cluster0.mongodb.net +MONGO_PORT=27017 +APP_NAME=my-app + +# Note: The connection string will be constructed as: +# mongodb://:@:/ +# ?ssl=true&replicaSet=globaldb&retrywrites=false +# &maxIdleTimeMS=120000&appName=@@ diff --git a/dbdump_mongo/.gitignore b/dbdump_mongo/.gitignore new file mode 100644 index 0000000..d518a48 --- /dev/null +++ b/dbdump_mongo/.gitignore @@ -0,0 +1,15 @@ +# Environment configuration files (contain sensitive credentials) +.env +.env.* +!.env.example + +# Dump directories (can be large) +dumps/ +*.dump + +# OS files +.DS_Store +Thumbs.db + +# Logs +*.log diff --git a/dbdump_mongo/README.md b/dbdump_mongo/README.md new file mode 100644 index 0000000..9f05435 --- /dev/null +++ b/dbdump_mongo/README.md @@ -0,0 +1,219 @@ +# MongoDB Dump Utility + +This directory contains a standalone tool for backing up MongoDB databases from remote servers. +It manages multiple environment configurations for easy switching between prod, uat, and other environments. + +## Quick Start + +### First Run + +```bash +./db-dump.sh dump +``` + +The script will detect no environments and guide you through creating one. Answer the prompts with: +- **Environment name** (required): e.g., `prod`, `uat`, `staging` +- **MongoDB username** (required): The admin user +- **MongoDB password** (required): The admin password +- **MongoDB host** (required): e.g., `cluster0.mongodb.net` +- **MongoDB port** (required): Usually `27017` or `10255` +- **Application name** (required): Used in connection string and dump folder name + +### Subsequent Runs + +```bash +./db-dump.sh dump +``` + +The script will show you a list of configured environments and let you choose which one to backup, +or create a new environment. + +## Commands + +```bash +# Perform a backup of a selected environment +./db-dump.sh dump + +# List all configured environments +./db-dump.sh list + +# Add a new environment +./db-dump.sh add + +# Delete an environment +./db-dump.sh delete + +# Show help +./db-dump.sh help +``` + +## How It Works + +1. **Environment Selection**: On each run, you select from existing environments or create new ones +2. **Credential Storage**: Each environment's credentials are stored in `.env.` files +3. **MongoDB Connection**: Connects using your credentials with MongoDB Atlas parameters (ssl, replicaSet, etc.) +4. **Dump Creation**: Uses `mongodump` to create a backup +5. **Timestamped Output**: Dumps are organized as `./dumps///` + +## Configuration Files + +Each environment is stored in a separate file: + +``` +.env.prod # Production environment +.env.uat # UAT environment +.env.staging # Staging environment (if created) +``` + +Each file contains: +``` +MONGO_USERNAME= +MONGO_PASSWORD= +MONGO_HOST= +MONGO_PORT= +APP_NAME= +``` + +## Directory Structure + +``` +dbdump/ +├── db-dump.sh # Main script +├── .env.prod # Production config (created by user) +├── .env.uat # UAT config (created by user) +├── .env.example # Example configuration +├── .gitignore # Git ignore rules +├── README.md # This file +└── dumps/ # Created automatically + ├── prod/ + │ ├── 20260414_1632/ + │ │ ├── admin/ + │ │ └── myapp/ + │ └── 20260414_1645/ + └── uat/ + └── 20260414_1632/ +``` + +## Dump Output + +Dumps are stored with the following structure: + +``` +dumps/// + admin/ # System databases + config/ + myapp/ # Application databases + system.profile/ +``` + +Each backup preserves the complete database structure and is ready to be restored using `mongorestore`. + +## Accessing Dumps + +To view or restore a dump: + +```bash +# List recent dumps +ls -lh dumps/prod/ + +# Restore a dump to local MongoDB +mongorestore --uri "mongodb://localhost:27017" dumps/prod/20260414_1632/ + +# Copy dump to another location +cp -r dumps/prod/20260414_1632/ /backup/prod-backup-20260414/ +``` + +## Security + +- Configuration files have restricted permissions (600) +- Credentials are stored locally in `.env.*` files +- These files are git-ignored for security +- Never commit `.env.*` files to version control + +## Requirements + +- **MongoDB Database Tools**: Download from https://www.mongodb.com/try/download/database-tools + - Specifically requires `mongodump` command +- Bash shell +- Network access to MongoDB server + +## Installation of MongoDB Database Tools + +### macOS +```bash +brew tap mongodb/brew +brew install mongodb-database-tools +``` + +### Linux (Ubuntu/Debian) +```bash +wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2004-x86_64-100.8.0.tgz +tar -xzf mongodb-database-tools-ubuntu2004-x86_64-100.8.0.tgz +sudo cp mongodb-database-tools-ubuntu2004-x86_64-100.8.0/bin/* /usr/local/bin/ +``` + +### Windows +Download the MSI installer from https://www.mongodb.com/try/download/database-tools and run it. + +## Troubleshooting + +### mongodump not found +```bash +# Check if MongoDB Database Tools is installed +mongodump --version + +# If not installed, see "Installation" section above +``` + +### Connection refused +- Verify host and port are correct +- Check network connectivity to MongoDB server +- Ensure firewall allows access + +### Authentication failed +- Verify username and password +- Check the user has admin permissions +- Confirm the `authSource=admin` parameter + +### Port already displaying in error +The connection string is checked for errors. If you see connection errors, the script displays the exact MongoDB error message. + +## Advanced Usage + +### Manual mongodump command (for reference) + +The script constructs and runs this command automatically: + +```bash +mongodump --uri "mongodb://:@:/?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@@" -o ./dumps// +``` + +### Scheduling automated backups + +You can schedule regular backups using cron (Linux/macOS): + +```bash +# Add to crontab +crontab -e + +# Daily dump at 3 AM for production environment +0 3 * * * cd /path/to/dbdump && ./db-dump.sh dump << EOF +1 +EOF +``` + +## Support + +For MongoDB connection issues, refer to: +- MongoDB Documentation: https://docs.mongodb.com/ +- Connection String Reference: https://docs.mongodb.com/manual/reference/connection-string/ + +## About + +**Author:** Julien Gautier ([jgautier.webdev@gmail.com](mailto:jgautier.webdev@gmail.com)) + +**Organization:** AdAstra + +**Project:** adastra_scripts + +This utility was created to enable efficient backup and versioning of MongoDB databases from remote servers for AdAstra projects. diff --git a/dbdump_mongo/db-dump.sh b/dbdump_mongo/db-dump.sh new file mode 100755 index 0000000..0755581 --- /dev/null +++ b/dbdump_mongo/db-dump.sh @@ -0,0 +1,450 @@ +#!/bin/bash + +# Author: Julien Gautier +# Organization: AdAstra +# Project: adastra_scripts +# +################################################################################ +# MongoDB Dump Utility Script (Standalone) +# +# This script performs backups (dumps) of MongoDB databases from remote or local servers. +# It manages multiple environments (prod, uat, etc.) with stored credentials. +# +# Usage: ./db-dump.sh +# +# Commands: +# dump Select environment and perform a dump +# list List all configured environments +# add Add a new environment +# delete Delete an environment +# help Show this help message +# +# Examples: +# ./db-dump.sh dump +# ./db-dump.sh list +# ./db-dump.sh add +# +# Configuration: +# Each environment is stored in a separate .env. file. +# All dumps are stored in ./dumps/// +# +################################################################################ + +set -e + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DUMPS_DIR="${SCRIPT_DIR}/dumps" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' + +# Helper functions +print_info() { + echo -e "${BLUE}┣━${NC} ${CYAN}${1}${NC}" +} + +print_success() { + echo -e "${GREEN}✓ ${1}${NC}" +} + +print_error() { + echo -e "${RED}✗ ${1}${NC}" +} + +print_warning() { + echo -e "${YELLOW}⚠ ${1}${NC}" +} + +print_separator() { + echo -e "${BLUE}┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +} + +print_separator_up() { + echo -e "${BLUE}┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +} + +print_separator_down() { + echo -e "${BLUE}┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +} + +print_section() { + echo + print_separator_up + print_info "$1" + print_separator_down + echo +} + +check_mongodump() { + if ! command -v mongodump &> /dev/null; then + print_error "mongodump is not installed or not in PATH" + print_info "Install MongoDB Database Tools from: https://www.mongodb.com/try/download/database-tools" + exit 1 + fi + print_success "mongodump found: $(mongodump --version)" +} + +get_environments() { + local envs=() + for env_file in "$SCRIPT_DIR"/.env.*; do + if [[ -f "$env_file" ]]; then + local env_name=$(basename "$env_file" | sed 's/^\.env\.//') + # Exclude the example file + if [[ "$env_name" != "example" ]]; then + envs+=("$env_name") + fi + fi + done + echo "${envs[@]}" +} + +env_file_path() { + echo "${SCRIPT_DIR}/.env.$1" +} + +create_environment() { + print_section "Create New Environment" + + # Check if environment name already exists + read -p "Enter environment name (prod, uat, etc.): " env_name + env_name="${env_name,,}" # Convert to lowercase + + if [[ -z "$env_name" ]]; then + print_error "Environment name cannot be empty" + return 1 + fi + + if [[ -f "$(env_file_path "$env_name")" ]]; then + print_error "Environment '$env_name' already exists" + return 1 + fi + + print_info "Enter MongoDB connection details for '$env_name'" + echo + + # Read credentials (no defaults) + read -p "MongoDB username: " mongo_username + if [[ -z "$mongo_username" ]]; then + print_error "Username cannot be empty" + return 1 + fi + + read -sp "MongoDB password: " mongo_password + echo + if [[ -z "$mongo_password" ]]; then + print_error "Password cannot be empty" + return 1 + fi + + read -p "MongoDB host (e.g., cluster0.mongodb.net): " mongo_host + if [[ -z "$mongo_host" ]]; then + print_error "Host cannot be empty" + return 1 + fi + + read -p "MongoDB port (e.g., 27017): " mongo_port + if [[ -z "$mongo_port" ]]; then + print_error "Port cannot be empty" + return 1 + fi + + read -p "Application name (appName): " app_name + if [[ -z "$app_name" ]]; then + print_error "Application name cannot be empty" + return 1 + fi + + # Create .env file for this environment + local env_file=$(env_file_path "$env_name") + cat > "$env_file" << EOF +# Environment: $env_name +# Created: $(date) +MONGO_USERNAME=${mongo_username} +MONGO_PASSWORD=${mongo_password} +MONGO_HOST=${mongo_host} +MONGO_PORT=${mongo_port} +APP_NAME=${app_name} +EOF + chmod 600 "$env_file" # Restrict permissions for security + + print_success "Environment '$env_name' created successfully" + print_info "Configuration stored in: $(basename "$env_file")" +} + +# Global variable to hold selected environment +SELECTED_ENVIRONMENT="" + +select_environment() { + SELECTED_ENVIRONMENT="" + local envs=($(get_environments)) + local count=${#envs[@]} + + print_separator_up + + if [[ $count -eq 0 ]]; then + print_info "No Environments Configured" + print_separator + echo + print_warning "No existing environments found" + echo + echo " 1) Create a new environment" + echo " 2) Exit" + echo + echo -n "Enter your choice (1-2): " + read choice + + if [[ "$choice" == "1" ]]; then + if create_environment; then + # After creating an environment, recursively call to select it + select_environment + return $? + else + return 1 + fi + else + return 1 + fi + fi + + print_info "Select Environment" + print_separator_down + echo + + for i in "${!envs[@]}"; do + echo " $((i + 1))) ${envs[$i]}" + done + echo " $((count + 1))) Create new environment" + echo + echo -n "Enter your choice (1-$((count + 1))): " + read choice + + if [[ "$choice" =~ ^[0-9]+$ ]] && [[ $choice -ge 1 ]] && [[ $choice -le $((count + 1)) ]]; then + if [[ $choice -eq $((count + 1)) ]]; then + if create_environment; then + # After creating an environment, recursively call to select it + select_environment + return $? + else + return 1 + fi + else + SELECTED_ENVIRONMENT="${envs[$((choice - 1))]}" + return 0 + fi + else + print_error "Invalid choice" + return 1 + fi +} + +load_environment() { + local env_name="$1" + local env_file=$(env_file_path "$env_name") + + if [[ ! -f "$env_file" ]]; then + print_error "Environment file not found: $env_file" + return 1 + fi + + # shellcheck source=/dev/null + source "$env_file" +} + +perform_dump() { + local env_name="$1" + + load_environment "$env_name" || return 1 + + print_section "MongoDB Dump: $env_name" + + # Create output directory with timestamp + local timestamp=$(date +%Y%m%d_%H%M) + local output_dir="${DUMPS_DIR}/${APP_NAME}/${timestamp}" + + mkdir -p "$output_dir" + + print_separator_up + print_info "Environment: $env_name" + print_info "App Name: $APP_NAME" + print_info "Host: $MONGO_HOST:$MONGO_PORT" + print_info "Output: $output_dir" + print_separator_down + + # Construct connection string + local connection_uri="mongodb://${MONGO_USERNAME}:${MONGO_PASSWORD}@${MONGO_HOST}:${MONGO_PORT}/?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@${APP_NAME}@" + + # Perform dump + if mongodump --uri "$connection_uri" -o "$output_dir" 2>&1; then + print_success "Dump completed successfully" + print_info "Output directory: $output_dir" + + # Show dump summary + local dump_size=$(du -sh "$output_dir" | cut -f1) + print_info "Total size: $dump_size" + else + print_error "Dump failed" + print_warning "Removing incomplete dump directory..." + rm -rf "$output_dir" + return 1 + fi +} + +cmd_dump() { + print_separator_up + print_info "MongoDB Dump Utility" + print_separator_down + + check_mongodump + + select_environment + if [[ $? -ne 0 ]]; then + print_error "No environment selected" + return 1 + fi + + if perform_dump "$SELECTED_ENVIRONMENT"; then + print_success "Dump operation completed" + else + print_error "Dump operation failed" + return 1 + fi +} + +cmd_list() { + print_section "Configured Environments" + + local envs=($(get_environments)) + + if [[ ${#envs[@]} -eq 0 ]]; then + print_warning "No environments configured yet" + return 0 + fi + + for env in "${envs[@]}"; do + local env_file=$(env_file_path "$env") + # shellcheck source=/dev/null + source "$env_file" + echo " • $env" + echo " Host: $MONGO_HOST:$MONGO_PORT" + echo " App: $APP_NAME" + echo + done +} + +cmd_add() { + if create_environment; then + print_success "Environment added successfully" + else + print_error "Failed to add environment" + return 1 + fi +} + +cmd_delete() { + print_section "Delete Environment" + + local envs=($(get_environments)) + + if [[ ${#envs[@]} -eq 0 ]]; then + print_warning "No environments configured" + return 0 + fi + + for i in "${!envs[@]}"; do + echo " $((i + 1))) ${envs[$i]}" + done + echo + + read -p "Enter environment number to delete: " choice + + if [[ "$choice" =~ ^[0-9]+$ ]] && [[ $choice -ge 1 ]] && [[ $choice -le ${#envs[@]} ]]; then + local env_name="${envs[$((choice - 1))]}" + read -p "Confirm deletion of '$env_name'? (yes/no): " confirm + + if [[ "$confirm" == "yes" ]]; then + local env_file=$(env_file_path "$env_name") + rm -f "$env_file" + print_success "Environment '$env_name' deleted" + else + print_info "Deletion cancelled" + fi + else + print_error "Invalid choice" + return 1 + fi +} + +cmd_help() { + cat << EOF + +MongoDB Dump Utility Script (Standalone) + +Usage: ./db-dump.sh + +Commands: + dump Select an environment and perform a backup dump + list List all configured environments + add Add a new environment configuration + delete Delete an environment configuration + help Show this help message + +Examples: + ./db-dump.sh dump # Interactive environment selection and dump + ./db-dump.sh list # Show all environments + ./db-dump.sh add # Add new environment + ./db-dump.sh delete # Remove an environment + +Environment Configuration: + - Each environment is stored in a separate .env. file + - Credentials are encrypted (file permissions: 600) + - Supported environments: prod, uat, dev, staging, etc. + +Dump Output: + - Dumps are stored in: ./dumps/// + - Each dump is timestamped for easy identification + - Connection string includes MongoDB Atlas parameters + +Requirements: + - MongoDB Database Tools (mongodump) + - Bash shell + +MongoDB Connection String Format: + mongodb://:@:/ + ?ssl=true&replicaSet=globaldb&retrywrites=false + &maxIdleTimeMS=120000&appName=@@ + +EOF +} + +# Main +COMMAND="${1:-help}" + +case "$COMMAND" in + dump) + cmd_dump + ;; + list) + cmd_list + ;; + add) + cmd_add + ;; + delete) + cmd_delete + ;; + help|--help|-h) + cmd_help + ;; + *) + print_error "Unknown command: $COMMAND" + echo + cmd_help + exit 1 + ;; +esac