-
-
Save leelakrishna/f90264365b892ef65203 to your computer and use it in GitHub Desktop.
HDFS Caching
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives | |
Found 0 entries | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives stats | |
Can't understand argument: stats | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats | |
Found 0 entries | |
kawasaki@hadoop11:~$ hadoop fs -ls dir1 | |
Found 4 items | |
drwxr-xr-x - kawasaki kawasaki 0 2014-04-09 06:38 dir1/a | |
-rw-r--r-- 3 kawasaki kawasaki 75288655 2014-04-09 06:44 dir1/bigfile | |
-rw-r--r-- 3 kawasaki kawasaki 24 2014-04-09 03:40 dir1/small.txt | |
drwxr-xr-x - kawasaki kawasaki 0 2014-04-09 07:07 dir1/snapshot | |
kawasaki@hadoop11:~$ hdfs cacheadmin -addPool testpool -owner kawasaki -group kawasaki | |
AccessControlException: Access denied for user kawasaki. Superuser privilege is required | |
kawasaki@hadoop11:~$ sudo -u hdfs hdfs cacheadmin -addPool testpool -owner kawasaki -group kawasaki | |
Successfully added cache pool testpool. | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats | |
Found 0 entries | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listPools -stats | |
Found 1 result. | |
NAME OWNER GROUP MODE LIMIT MAXTTL BYTES_NEEDED BYTES_CACHED BYTES_OVERLIMIT FILES_NEEDED FILES_CACHED | |
testpool kawasaki kawasaki rwxr-xr-x unlimited never 0 0 0 0 0 | |
kawasaki@hadoop11:~$ hdfs cacheadmin -addDirective -path dir1/bigfile -pool testpool -force -replication 3 | |
Added cache directive 1 | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -statsFound 1 entry | |
ID POOL REPL EXPIRY PATH BYTES_NEEDED BYTES_CACHED FILES_NEEDED FILES_CACHED | |
1 testpool 3 never /user/kawasaki/dir1/bigfile 225865965 0 1 0 | |
kawasaki@hadoop11:~$ hadoop fs -get dir1/bigfile xxx | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats | |
Found 1 entry | |
ID POOL REPL EXPIRY PATH BYTES_NEEDED BYTES_CACHED FILES_NEEDED FILES_CACHED | |
1 testpool 3 never /user/kawasaki/dir1/bigfile 225865965 225865965 1 1 | |
kawasaki@hadoop11:~$ hadoop fs -get dir1/bigfile xxxx | |
kawasaki@hadoop11:~$ hadoop fs -put xxx dir1/bigfile2 | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats | |
Found 1 entry | |
ID POOL REPL EXPIRY PATH BYTES_NEEDED BYTES_CACHED FILES_NEEDED FILES_CACHED | |
1 testpool 3 never /user/kawasaki/dir1/bigfile 225865965 225865965 1 1 | |
(reverse-i-search)`pool': hdfs cacheadmin -addDirective -path dir1/bigfile -pool test^Col -force -replication 3 | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listPools -stats | |
Found 1 result. | |
NAME OWNER GROUP MODE LIMIT MAXTTL BYTES_NEEDED BYTES_CACHED BYTES_OVERLIMIT FILES_NEEDED FILES_CACHED | |
testpool kawasaki kawasaki rwxr-xr-x unlimited never 225865965 225865965 0 1 1 | |
kawasaki@hadoop11:~$ hdfs cacheadmin -addDirective -path dir1/bigfile2 -pool testpool -force -replication 3 | |
Added cache directive 2 | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listPools -statsFound 1 result. | |
NAME OWNER GROUP MODE LIMIT MAXTTL BYTES_NEEDED BYTES_CACHED BYTES_OVERLIMIT FILES_NEEDED FILES_CACHED | |
testpool kawasaki kawasaki rwxr-xr-x unlimited never 451731930 225865965 0 2 1 | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -statsFound 2 entries | |
ID POOL REPL EXPIRY PATH BYTES_NEEDED BYTES_CACHED FILES_NEEDED FILES_CACHED | |
1 testpool 3 never /user/kawasaki/dir1/bigfile 225865965 225865965 1 1 | |
2 testpool 3 never /user/kawasaki/dir1/bigfile2 225865965 0 1 0 | |
kawasaki@hadoop11:~$ time hadoop fs -get dir1/bigfile2 yyy | |
real 0m2.823s | |
user 0m3.560s | |
sys 0m0.828s | |
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats | |
Found 2 entries | |
ID POOL REPL EXPIRY PATH BYTES_NEEDED BYTES_CACHED FILES_NEEDED FILES_CACHED | |
1 testpool 3 never /user/kawasaki/dir1/bigfile 225865965 225865965 1 1 | |
2 testpool 3 never /user/kawasaki/dir1/bigfile2 225865965 225865965 1 1 | |
kawasaki@hadoop11:~$ hadoop fs -get dir1/bigfile2 zzzkawasaki@hadoop11:~$ time hadoop fs -get dir1/bigfile2 zzzz | |
real 0m2.822s | |
user 0m3.621s | |
sys 0m0.804s | |
kawasaki@hadoop11:~$ hadoop fs -put zzzz dir1/bigfile3 | |
kawasaki@hadoop11:~$ time hadoop fs -get dir1/bigfile3 | |
real 0m2.698s | |
user 0m3.632s | |
sys 0m0.694s |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment