HDFS Caching

gistfile1.txt

kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives
Found 0 entries
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives stats
Can't understand argument: stats
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats
Found 0 entries
kawasaki@hadoop11:~$ hadoop fs -ls dir1
Found 4 items
drwxr-xr-x   - kawasaki kawasaki          0 2014-04-09 06:38 dir1/a
-rw-r--r--   3 kawasaki kawasaki   75288655 2014-04-09 06:44 dir1/bigfile
-rw-r--r--   3 kawasaki kawasaki         24 2014-04-09 03:40 dir1/small.txt
drwxr-xr-x   - kawasaki kawasaki          0 2014-04-09 07:07 dir1/snapshot
kawasaki@hadoop11:~$ hdfs cacheadmin -addPool testpool -owner kawasaki -group kawasaki 
AccessControlException: Access denied for user kawasaki. Superuser privilege is required
kawasaki@hadoop11:~$ sudo -u hdfs hdfs cacheadmin -addPool testpool -owner kawasaki -group kawasaki 
Successfully added cache pool testpool.
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats
Found 0 entries
kawasaki@hadoop11:~$ hdfs cacheadmin -listPools -stats
Found 1 result.
NAME      OWNER     GROUP     MODE            LIMIT  MAXTTL  BYTES_NEEDED  BYTES_CACHED  BYTES_OVERLIMIT  FILES_NEEDED  FILES_CACHED
testpool  kawasaki  kawasaki  rwxr-xr-x   unlimited   never             0             0                0             0             0
kawasaki@hadoop11:~$ hdfs cacheadmin -addDirective -path dir1/bigfile -pool testpool -force -replication 3
Added cache directive 1
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -statsFound 1 entry
 ID POOL       REPL EXPIRY  PATH                          BYTES_NEEDED  BYTES_CACHED  FILES_NEEDED  FILES_CACHED
  1 testpool      3 never   /user/kawasaki/dir1/bigfile      225865965             0             1             0
kawasaki@hadoop11:~$ hadoop fs -get dir1/bigfile xxx
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats
Found 1 entry
 ID POOL       REPL EXPIRY  PATH                          BYTES_NEEDED  BYTES_CACHED  FILES_NEEDED  FILES_CACHED
  1 testpool      3 never   /user/kawasaki/dir1/bigfile      225865965     225865965             1             1
kawasaki@hadoop11:~$ hadoop fs -get dir1/bigfile xxxx
kawasaki@hadoop11:~$ hadoop fs -put xxx dir1/bigfile2
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats
Found 1 entry
 ID POOL       REPL EXPIRY  PATH                          BYTES_NEEDED  BYTES_CACHED  FILES_NEEDED  FILES_CACHED
  1 testpool      3 never   /user/kawasaki/dir1/bigfile      225865965     225865965             1             1
(reverse-i-search)`pool': hdfs cacheadmin -addDirective -path dir1/bigfile -pool test^Col -force -replication 3
kawasaki@hadoop11:~$ hdfs cacheadmin -listPools -stats
Found 1 result.
NAME      OWNER     GROUP     MODE            LIMIT  MAXTTL  BYTES_NEEDED  BYTES_CACHED  BYTES_OVERLIMIT  FILES_NEEDED  FILES_CACHED
testpool  kawasaki  kawasaki  rwxr-xr-x   unlimited   never     225865965     225865965                0             1             1
kawasaki@hadoop11:~$ hdfs cacheadmin -addDirective -path dir1/bigfile2 -pool testpool -force -replication 3
Added cache directive 2
kawasaki@hadoop11:~$ hdfs cacheadmin -listPools -statsFound 1 result.
NAME      OWNER     GROUP     MODE            LIMIT  MAXTTL  BYTES_NEEDED  BYTES_CACHED  BYTES_OVERLIMIT  FILES_NEEDED  FILES_CACHED
testpool  kawasaki  kawasaki  rwxr-xr-x   unlimited   never     451731930     225865965                0             2             1
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -statsFound 2 entries
 ID POOL       REPL EXPIRY  PATH                           BYTES_NEEDED  BYTES_CACHED  FILES_NEEDED  FILES_CACHED
  1 testpool      3 never   /user/kawasaki/dir1/bigfile       225865965     225865965             1             1
  2 testpool      3 never   /user/kawasaki/dir1/bigfile2      225865965             0             1             0
kawasaki@hadoop11:~$ time hadoop fs -get dir1/bigfile2 yyy 
real	0m2.823s
user	0m3.560s
sys	0m0.828s
kawasaki@hadoop11:~$ hdfs cacheadmin -listDirectives -stats
Found 2 entries
 ID POOL       REPL EXPIRY  PATH                           BYTES_NEEDED  BYTES_CACHED  FILES_NEEDED  FILES_CACHED
  1 testpool      3 never   /user/kawasaki/dir1/bigfile       225865965     225865965             1             1
  2 testpool      3 never   /user/kawasaki/dir1/bigfile2      225865965     225865965             1             1
kawasaki@hadoop11:~$ hadoop fs -get dir1/bigfile2 zzzkawasaki@hadoop11:~$ time hadoop fs -get dir1/bigfile2 zzzz

real	0m2.822s
user	0m3.621s
sys	0m0.804s
kawasaki@hadoop11:~$ hadoop fs -put zzzz dir1/bigfile3
kawasaki@hadoop11:~$ time hadoop fs -get dir1/bigfile3

real	0m2.698s
user	0m3.632s
sys	0m0.694s