########################### SQOOP PRACTICE ######################################

##copy the mysql jdbc connector to VM machine desktop

##Extract the mysql-connector-java-5.1.39.tar.gz

##copy the mysql-connector-java-5.1.39-bin.jar from extracted folder to /usr/lib/sqoop/lib

##to copy it we need super user login using sudo

[~]$ sudo cp Desktop/mysql-connector-java-5.1.39/mysql-connector-java-5.1.39-bin.jar

/usr/lib/sqoop/lib/

##login to mysql ****** Copy Hemp & Hdept to /tmp

[cloudera@localhost ~]$ mysql --user root

##create a database

show databases;

CREATE DATABASE sqoop_db;

show databases;

use sqoop_db;

show tables;

###create table employees for the tab delimited file Hemp.txt available in /tmp
****************MYSQL****************************
CREATE TABLE employees
(empid int,
name varchar(15),
salary int,
deptid int);
CREATE TABLE departments
(deptid int,
deptname varchar(15));
--copy file Hemp & Hdept from desktop to /tmp
cp Desktop/Hemp.txt /tmp/
cp Desktop/Hdept.txt /tmp/

##Load data from /tmp/Hemp.txt into table employees

LOAD DATA INFILE '/tmp/Hemp.txt' INTO TABLE employees;

##Load data from /tmp/Hdept.txt into table departments

LOAD DATA INFILE '/tmp/Hdept.txt' INTO TABLE departments;

SELECT * FROM employees;

SELECT * FROM departments;

############################################
create following tables

create table emp as select * from employees limit 5;

create table emp1 as select * from employees limit 10;

create table emp2 as select * from employees limit 15;

##Grant all privileges to any user on localhost on sqoop db

GRANT ALL PRIVILEGES ON sqoop_db TO ''@'localhost';

##Now lets transfer data from mysql employees table to HDFS , default directory i.e. directory
created by sqoop with table name

**********************************************************************************
hadoop fs -ls

sqoop import --connect jdbc:mysql://localhost/sqoop_db --table employees --username root -m 1

##now confirm the transfer in hdfs

hadoop fs -ls ##we will see a directory named employees as sqoop will create one with the table
name

hadoop fs -ls employees ##we will see 3 files

hadoop fs -cat employees/part-m-00000 ##file containing data of employees table from

mysql db

**********************************************************************************
****************************
Now lets transfer data from employees table to HDFS , a explicit directory name mentioned by us
(the name we mention should not be a existing directory name)
**********************************************************************************
****************************

sqoop
import
--connect jdbc:mysql://localhost/sqoop_db
--table employees
--username root
--target-dir msqp_dir/emp
-m 1

hadoop fs -ls

hadoop fs -ls msqp_dir

hadoop fs -ls msqp_dir/emp

hadoop fs -cat msqp_dir/emp/part-m-00000

**********************************************************************************
****************************
Now lets transfer data from any table to HDFS , a explicit warehouse directory name mentioned by
us
We ca use the same warehouse directory name for all sqoop table imports from mysql to hdfs
By default, Sqoop will create a directory with the same name as the imported table inside your home
directory on HDFS and import all data there
**********************************************************************************
**************************** warehouse-dir is resusable directory

hadoop fs -ls

sqoop import --connect jdbc:mysql://localhost/sqoop_db --table employees --username root

--warehouse-dir sq_dir
-m 1

sqoop import --connect jdbc:mysql://localhost/sqoop_db --table departments --username root

--warehouse-dir sq_dir -m 1

**********************************************************************************
****************************
Now lets transfer all tables from mysql sqoop_db to HDFS , delete any existing same name
directories as table names from hdfs location
**********************************************************************************
****************************

mysql > show tables;

hadoop fs -rmr (employees,emp,departments remove the directories)

sqoop import-all-tables --connect jdbc:mysql://localhost/sqoop_db --username root -m 1

**********************************************************************************
****************
Now lets transfer all rows with salary < 5000 from employees to HDFS
**********************************************************************************
*****************
sqoop import --connect jdbc:mysql://localhost/sqoop_db --table employees
--where 'salary<5000'
--target-dir empsal
--username root -m 1
**********************************************************************************
****************************
Now lets transfer only empid,name,deptid columns from employees to HDFS
**********************************************************************************
***************************
sqoop import --connect jdbc:mysql://localhost/sqoop_db --table employees
--columns "empid,name,deptid"
--target-dir empcols
--username root -m 1

**********************************************************************************
****************************
spilt the employees data in mysql table into 4 files on hdfs while importing using sqoop
**********************************************************************************
****************************
sqoop import --connect jdbc:mysql://localhost/sqoop_db
--table employees
--warehouse-dir sq_dir /empnew
--username root
--split-by empid
-m 4

**********************************************************************************
***************************
create only table structure in hive database based on mysql table
**********************************************************************************
****************************

sqoop
create-hive-table
--connect jdbc:mysql://localhost/sqoop_db
--table emp1
--username root
**********************************************************************************
****************************
create and import data same time in hive db from mysql table
**********************************************************************************
****************************

sqoop
import
--connect jdbc:mysql://localhost/sqoop_db
--table emp2
--username root
--hive-import
-m 1

**********************************************************************************
****************************
create and import data same time in hive db from mysql table, default delimiter is none and hence
we mention a delimiter explicitly
**********************************************************************************
****************************

sqoop import
--connect jdbc:mysql://localhost/sqoop_db
--table departments
--username root
--hive-import
--fields-terminated-by '@'
-m 1

**********************************************************************************
****************************

Export data from hive table data to mysql table using sqoop
**********************************************************************************
****************************

##create table empx in mysql

mysql>
mysql> create table deptx as select * from departments where 1=0;
mysql> select * from deptx;
##export data from hive warehouse directory
=======Transfering data from hive table to existing table in mysql====

sqoop export
--connect jdbc:mysql://localhost/sqoop_db
--table deptx
--export-dir '/user/hive/warehouse/departments'
--fields-terminated-by '@'
--username root

##Confirm table in mysql for data

mysql> select * from deptx;