Scribd
Upload
11 views5 pages

Aws Setup

This Bash script automates the setup of a Docker environment with ZFS storage, including the installation of Docker, ZFS utilities, and Docker Compose. It configures a ZFS pool for Docker, creates a project directory, and sets up a Docker Compose file for various services like Prometheus, Grafana, and PostgreSQL. Additionally, it includes a database interface script for managing a PostgreSQL database and uploading CSV data.

Uploaded by

yomaxi5211
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as TXT, PDF, TXT or read online on Scribd
11 views5 pages

Aws Setup

This Bash script automates the setup of a Docker environment with ZFS storage, including the installation of Docker, ZFS utilities, and Docker Compose. It configures a ZFS pool for Docker, creates a project directory, and sets up a Docker Compose file for various services like Prometheus, Grafana, and PostgreSQL. Additionally, it includes a database interface script for managing a PostgreSQL database and uploading CSV data.

Uploaded by

yomaxi5211
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as TXT, PDF, TXT or read online on Scribd
You are on page 1/ 5

#!

/bin/bash

sudo apt update && sudo apt upgrade -y

# Ensure the script is running as root


if [ "$(id -u)" -ne 0 ]; then
echo "This script must be run as root"
exit 1
fi

# Check if the virtual disks xvdb and xvdc are correctly configured
echo "Checking disks xvdb and xvdc..."
fdisk -l | grep -E "xvdb|xvdc"

# Install Docker and ZFS utilities


echo "Installing Docker and ZFS utilities..."
apt-get update && apt-get install -y docker.io zfsutils-linux curl

# Ensure no existing ZFS pools will conflict with our setup


echo "Checking for existing ZFS pools..."
zpool list

# Backup existing Docker data directory


echo "Backing up existing Docker data directory..."
cp -au /var/lib/docker /var/lib/docker.bk

# Stop Docker services to prevent data loss during setup


echo "Stopping Docker services..."
systemctl stop docker
systemctl stop docker.socket

# Remove old Docker data directory to ensure a clean state


echo "Removing old Docker data directory..."
rm -rf /var/lib/docker

# Create a mirrored ZFS pool named 'Docker', mounted at '/var/lib/docker'


echo "Creating ZFS pool named 'Docker' with disks xvdb and xvdc..."
zpool create Docker -m /var/lib/docker mirror xvdb xvdc

# Configure Docker to use ZFS as the storage driver


echo "Configuring Docker to use ZFS as the storage driver..."
echo '{"storage-driver": "zfs"}' | tee /etc/docker/daemon.json

# Restart Docker to apply the new storage configuration


echo "Restarting Docker services..."
service docker stop
service docker start

# Verify that Docker is using ZFS


echo "Verifying Docker configuration..."
docker info

# Test Docker setup by pulling an image and checking ZFS pool space usage
echo "Testing Docker with ZFS by pulling an image..."
docker pull pihole/pihole
docker images
zpool list
# Clean up Docker images and stop Docker if needed
echo "Cleaning up: stopping Docker and removing test image..."
systemctl stop docker
docker images
docker rmi pihole/pihole

# Install Docker Compose


echo "Installing Docker Compose..."
curl -L "https://github.com/docker/compose/releases/download/v2.15.0/docker-
compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose

# Verify Docker Compose installation


if ! command -v docker-compose &> /dev/null; then
echo "docker-compose could not be installed"
exit 1
fi

# Prompt for user input


read -p "Enter the username: " username
read -p "Enter the IP address for enX0: " ip_address

# Set IP address for enX0


ip addr add $ip_address dev enX0

# Create project directory


mkdir -p /home/$username/Blockchain_prj
cd /home/$username/Blockchain_prj

# Move CSV file to project directory


if [ -f "/home/$username/Downloads/yellow_tripdata_2024.csv" ]; then
mv "/home/$username/Downloads/yellow_tripdata_2024.csv"
"/home/$username/Blockchain_prj/"
echo "Moved yellow_tripdata_2024.csv to /home/$username/Blockchain_prj/"
else
echo "yellow_tripdata_2024.csv not found. Please ensure the file exists."
exit 1
fi

# Create docker-compose.yml file


cat > docker-compose.yml <<EOF
services:
prometheus:
image: prom/prometheus:latest
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
ports:
- "9090:9090"
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--web.enable-lifecycle'

node-exporter:
image: prom/node-exporter:latest
ports:
- "9100:9100"
volumes:
- /:/host:ro,rslave
command:
- '--path.rootfs=/host'
- '--collector.zfs'
restart: always

grafana:
image: grafana/grafana:latest
ports:
- "3000:3000"
depends_on:
- prometheus

db:
image: postgres
restart: always
environment:
POSTGRES_PASSWORD: example
ports:
- "5432:5432"

pgadmin:
image: dpage/pgadmin4
restart: always
environment:
PGADMIN_DEFAULT_EMAIL: [email protected]
PGADMIN_DEFAULT_PASSWORD: example
ports:
- "8080:80"
depends_on:
- db
EOF

# Create prometheus.yml file


cat > prometheus.yml <<EOF
global:
scrape_interval: 15s

scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['$ip_address:9090']

- job_name: 'node-exporter'
static_configs:
- targets: ['node-exporter:9100']
EOF

# Create database_interface.sh script


cat > database_interface.sh <<'EOF'
#!/bin/bash
DB_USER="postgres"
DB_PASSWORD="example"
DB_NAME="yellow_tripdata_2024"
CSV_FILE="yellow_tripdata_2024.csv"
DOCKER_SERVICE="db"

create_db_and_table() {
docker-compose exec $DOCKER_SERVICE psql -U $DB_USER -c "CREATE DATABASE
$DB_NAME;" || true
docker-compose exec $DOCKER_SERVICE psql -U $DB_USER -d $DB_NAME -c "
CREATE TABLE IF NOT EXISTS yellow_tripdata_2024 (
VendorID INTEGER,
tpep_pickup_datetime TIMESTAMP,
tpep_dropoff_datetime TIMESTAMP,
passenger_count DOUBLE PRECISION,
trip_distance DOUBLE PRECISION,
RatecodeID DOUBLE PRECISION,
store_and_fwd_flag TEXT,
PULocationID INTEGER,
DOLocationID INTEGER,
payment_type BIGINT,
fare_amount DOUBLE PRECISION,
extra DOUBLE PRECISION,
mta_tax DOUBLE PRECISION,
tip_amount DOUBLE PRECISION,
tolls_amount DOUBLE PRECISION,
improvement_surch DOUBLE PRECISION,
total_amount DOUBLE PRECISION,
congestion_surcharge DOUBLE PRECISION,
Airport_fee DOUBLE PRECISION
);"
}

upload_csv() {
docker-compose cp $CSV_FILE $DOCKER_SERVICE:/tmp/$CSV_FILE
docker-compose exec $DOCKER_SERVICE psql -U $DB_USER -d $DB_NAME -c "\COPY
yellow_tripdata_2024 FROM '/tmp/$CSV_FILE' WITH (FORMAT csv, HEADER true);"
}

delete_data() {
docker-compose exec $DOCKER_SERVICE psql -U $DB_USER -d $DB_NAME -c "DELETE
FROM yellow_tripdata_2024;"
}

show_menu() {
echo "1) Create Database and Table"
echo "2) Upload CSV"
echo "3) Delete Data"
echo "4) Exit"
}

while true; do
show_menu
read -p "Enter your choice: " choice
case $choice in
1) create_db_and_table ;;
2) upload_csv ;;
3) delete_data ;;
4) exit 0 ;;
*) echo "Invalid option" ;;
esac
done
EOF

chmod +x database_interface.sh

# Start Docker services


docker-compose up -d
# Run the database interface script
/bin/bash database_interface.sh

echo "Setup completed. Please verify the configuration."

You might also like