Hadoop Fs Commands

Hadoop.sh

hadoop fs -put /data/retail_db/order_items/part-00000 /user/koushikmln/retail_db_order_items.csv

#Set Blocksize
hadoop fs -D dfs.blocksize=67108864 -put /data/retail_db/order_items/part-00000 /user/koushikmln/retail_db_order_items.csv

#Set Replication Factor and Black Size
hadoop fs -D dfs.blocksize=67108864 -D dfs.replication=1 -put /data/retail_db/order_items/part-00000 /user/koushikmln/retail_db_order_items.csv

#Get File Metadata
hdfs fsck /user/koushikmln/retail_db_order_items.csv
hdfs fsck /user/koushikmln/retail_db_order_items.csv -files -blocks -locations
#We can monitor datanode capacity from name node ui http://nn01.itversity.com:50070/dfshealth.html#tab-datanode

#Delete Files
hadoop fs -rm /user/koushikmln/retail_db_order_items.csv

#List Files
hadoop fs -ls /user/koushikmln/
hadoop fs -ls -ltr /user/koushikmln/

#View File
hadoop fs -cat /user/koushikmln/retail_db_order_items.csv
hadoop fs -tail -f /user/koushikmln/retail_db_order_items.csv #Check the last few lines of a large file

#Updating a file is not possible in hadoop. We can only append to an existing file
hadoop fs -appendToFile test1.txt /user/koushikmln/test2.txt

#CopyFromLocal and CopyToLocal
hadoop fs -copyFromLocal /data/retail_db/order_items/part-00000 /user/koushikmln/retail_db_order_items.csv
hadoop fs -copyToLocal /user/koushikmln/retail_db_order_items.csv /home/koushikmln/retail_db_order_items.local.csv

#Permissions
hadoop fs  -chmod -R 750 /user/koushikmln/retail_db_order_items.csv
#Permissions owner,group,other -> r=4,w=2,execute=1 
#Permissions 750 => owner(read,write,execute), group(read,execute), others()

#chown command changes the owner of the files.Only super users can change the owners.Super user in hadoop is hdfs.
sudo -u hdfs hadoop fs -chown training /user/koushikmln/retail_db_order_items.csv

#Find
hadoop fs -find /user/koushikmln  -name orders

#DF, DU
hadoop fs -df -h #Display Hadoop Disk Usage
hadoop fs -du -sh /user/koushikmln #Display Files and Size and other Metadata