hadoopsters

ANALYZE TABLE my_database.my_table compute statistics for column1, column2, column3; -- column stats for non-partitioned table
ANALYZE TABLE my_database.my_table PARTITION (YEAR=2017, MONTH=11, DAY=30, HOUR=0) compute statistics for column1, column2, column3; -- column stats for single hour of partitioned table
ANALYZE TABLE my_database.my_table PARTITION (YEAR=2017, MONTH=11, DAY=30, HOUR) compute statistics for column1, column2, column3; -- column stats for a single day of partitioned table

hadoopsters

set hive.compute.query.using.stats=true;
set hive.stats.fetch.column.stats=true;

hadoopsters

CREATE TABLE my_database.my_table
STORED AS ORC TBLPROPERTIES('ORC.COMPRESS'='SNAPPY') as
SELECT * FROM my_database.my_other_table WHERE YEAR=2017 AND MONTH=11 AND DAY=30;

hadoopsters

select * from my_table
limit 10000;

hadoopsters

select * from my_table
order by rand()
limit 10000;

hadoopsters

select * from my_table
sort by rand()
limit 10000;

hadoopsters

select * from my_table
distribute by rand()
sort by rand()
limit 10000;

hadoopsters

select * from my_table
where rand() <= 0.0001
distribute by rand()
sort by rand()
limit 10000;

hadoopsters

#!/bin/bash
hive -e "insert overwrite local directory '/path/in/local/'
row format delimited fields terminated by ','
select * from my_database.my_table"
cat /path/in/local/* > /another/path/in/local/my_table.csv

hadoopsters

val conf = new SparkConf().setAppName(appName) // run on cluster
val ssc = new StreamingContext(conf, Seconds(5))
val sc = ssc.sparkContext
sc.setLogLevel("ERROR")