Aws Setup
Aws Setup
/bin/bash
# Check if the virtual disks xvdb and xvdc are correctly configured
echo "Checking disks xvdb and xvdc..."
fdisk -l | grep -E "xvdb|xvdc"
# Test Docker setup by pulling an image and checking ZFS pool space usage
echo "Testing Docker with ZFS by pulling an image..."
docker pull pihole/pihole
docker images
zpool list
# Clean up Docker images and stop Docker if needed
echo "Cleaning up: stopping Docker and removing test image..."
systemctl stop docker
docker images
docker rmi pihole/pihole
node-exporter:
image: prom/node-exporter:latest
ports:
- "9100:9100"
volumes:
- /:/host:ro,rslave
command:
- '--path.rootfs=/host'
- '--collector.zfs'
restart: always
grafana:
image: grafana/grafana:latest
ports:
- "3000:3000"
depends_on:
- prometheus
db:
image: postgres
restart: always
environment:
POSTGRES_PASSWORD: example
ports:
- "5432:5432"
pgadmin:
image: dpage/pgadmin4
restart: always
environment:
PGADMIN_DEFAULT_EMAIL: [email protected]
PGADMIN_DEFAULT_PASSWORD: example
ports:
- "8080:80"
depends_on:
- db
EOF
scrape_configs:
- job_name: 'prometheus'
static_configs:
- targets: ['$ip_address:9090']
- job_name: 'node-exporter'
static_configs:
- targets: ['node-exporter:9100']
EOF
create_db_and_table() {
docker-compose exec $DOCKER_SERVICE psql -U $DB_USER -c "CREATE DATABASE
$DB_NAME;" || true
docker-compose exec $DOCKER_SERVICE psql -U $DB_USER -d $DB_NAME -c "
CREATE TABLE IF NOT EXISTS yellow_tripdata_2024 (
VendorID INTEGER,
tpep_pickup_datetime TIMESTAMP,
tpep_dropoff_datetime TIMESTAMP,
passenger_count DOUBLE PRECISION,
trip_distance DOUBLE PRECISION,
RatecodeID DOUBLE PRECISION,
store_and_fwd_flag TEXT,
PULocationID INTEGER,
DOLocationID INTEGER,
payment_type BIGINT,
fare_amount DOUBLE PRECISION,
extra DOUBLE PRECISION,
mta_tax DOUBLE PRECISION,
tip_amount DOUBLE PRECISION,
tolls_amount DOUBLE PRECISION,
improvement_surch DOUBLE PRECISION,
total_amount DOUBLE PRECISION,
congestion_surcharge DOUBLE PRECISION,
Airport_fee DOUBLE PRECISION
);"
}
upload_csv() {
docker-compose cp $CSV_FILE $DOCKER_SERVICE:/tmp/$CSV_FILE
docker-compose exec $DOCKER_SERVICE psql -U $DB_USER -d $DB_NAME -c "\COPY
yellow_tripdata_2024 FROM '/tmp/$CSV_FILE' WITH (FORMAT csv, HEADER true);"
}
delete_data() {
docker-compose exec $DOCKER_SERVICE psql -U $DB_USER -d $DB_NAME -c "DELETE
FROM yellow_tripdata_2024;"
}
show_menu() {
echo "1) Create Database and Table"
echo "2) Upload CSV"
echo "3) Delete Data"
echo "4) Exit"
}
while true; do
show_menu
read -p "Enter your choice: " choice
case $choice in
1) create_db_and_table ;;
2) upload_csv ;;
3) delete_data ;;
4) exit 0 ;;
*) echo "Invalid option" ;;
esac
done
EOF
chmod +x database_interface.sh