mapping.sql

with tmp as (
  select
    -- group by is sometimes faster than distinct
    -- distinct extract_feature(feature) as feature
    extract_feature(feature) as feature
  from
    test l
    lateral view explode(features) r as feature
), 
mapped as (
  select
    feature,
    feature_hashing(feature) as index
  from
    tmp
  group by
    feature
)
-- INSERT OVERWRITE TABLE mapping
select
  index,
  collect_set(feature) as features -- collision can be happened
from 
  mapped
group by 
  index
-- order by index asc
-- limit 100

https://console.treasuredata.com/app/jobs/494047056/results