ingest.sh

time cat <<'EOF' | curl -XPOST --data-binary @- http://uranium.ics.uci.edu:19002/aql
drop dataverse twitter if exists;
create dataverse twitter if not exists;
use dataverse twitter

create type typeUser if not exists as open {
    id: int64,
    name: string,
    screen_name : string,
    lang : string,
    location: string,
    create_at: date,
    description: string,
    followers_count: int32,
    friends_count: int32,
    statues_count: int64
}

create type typePlace if not exists as open{
    country : string,
    country_code : string,
    full_name : string,
    id : string,
    name : string,
    place_type : string,
    bounding_box : rectangle
}

create type typeGeoTag if not exists as open {
    stateID: int32,
    stateName: string,
    countyID: int32,
    countyName: string,
    cityID: int32?,
    cityName: string?
}

create type typeTweet if not exists as open{
    create_at : datetime,
    id: int64,
    "text": string,
    in_reply_to_status : int64,
    in_reply_to_user : int64,
    favorite_count : int64,
    coordinate: point?,
    retweet_count : int64,
    lang : string,
    is_retweet: boolean,
    hashtags : {{ string }} ?,
    user_mentions : {{ int64 }} ? ,
    user : typeUser,
    place : typePlace?,
    geo_tag: typeGeoTag
}

create dataset ds_tweet(typeTweet) if not exists primary key id using compaction policy prefix (("max-mergable-component-size"="134217728"),("max-tolerance-component-count"="10")) with filter on create_at ;
// with filter on create_at;
//"using" "compaction" "policy" CompactionPolicy ( Configuration )? )?
create index text_idx if not exists on ds_tweet("text") type keyword;
//create index time_idx if not exists on ds_tweet(create_at) type btree;
//create index location_idx if not exists on ds_tweet(coordinate) type rtree;
//create index state_idx if not exists on ds_tweet(geo_tag.stateID) type btree;
//create index county_idx if not exists on ds_tweet(geo_tag.countyID) type btree;
//create index city_idx if not exists on ds_tweet(geo_tag.cityID) type btree;

//create feed MessageFeed using localfs(
//("path"="128.195.52.82:///home/jianfeng/data/828.h1w.adm"),
//("format"="adm"),
//("type-name"="typeTweet"));

//set wait-for-completion-feed "true";
//connect feed MessageFeed to dataset ds_tweet;

EOF