Skip to content

Instantly share code, notes, and snippets.

@cswinter
Created July 8, 2018 22:35
Show Gist options
  • Save cswinter/afea87df9ae293f0fd664d0d747759d1 to your computer and use it in GitHub Desktop.
Save cswinter/afea87df9ae293f0fd664d0d747759d1 to your computer and use it in GitHub Desktop.
m:(`trip_id ;"I";
`vendor_id ;"*";
`pickup_datetime ;"P";
`dropoff_datetime ;"P";
`store_and_fwd_flag ;"B";
`rate_code_id ;"X";
`pickup_longitude ;"E";
`pickup_latitude ;"E";
`dropoff_longitude ;"E";
`dropoff_latitude ;"E";
`passenger_count ;"X";
`trip_distance ;"E";
`fare_amount ;"E";
`extra ;"E";
`mta_tax ;"E";
`tip_amount ;"E";
`tolls_amount ;"E";
`ehail_fee ;"E";
`improvement_surcharge;"E";
`total_amount ;"E";
`payment_type ;"*";
`trip_type ;"X";
`pickup ;"*";
`dropoff ;"*";
`cab_type ;"*";
`precipitation ;"H";
`snow_depth ;"H";
`snowfall ;"H";
`max_temperature ;"H";
`min_temperature ;"H";
`average_wind_speed ;"H";
`pickup_nyct2010_gid ;"H";
`pickup_ctlabel ;"*";
`pickup_borocode ;"H";
`pickup_boroname ;"*";
`pickup_ct2010 ;"*";
`pickup_boroct2010 ;"*";
`pickup_cdeligibil ;"C";
`pickup_ntacode ;"*";
`pickup_ntaname ;"*";
`pickup_puma ;"H";
`dropoff_nyct2010_gid ;"H";
`dropoff_ctlabel ;"*";
`dropoff_borocode ;"H";
`dropoff_boroname ;"*";
`dropoff_ct2010 ;"*";
`dropoff_boroct2010 ;"*";
`dropoff_cdeligibil ;"C";
`dropoff_ntacode ;"*";
`dropoff_ntaname ;"*";
`dropoff_puma ;"H")
@[`.;`c`t;:;flip 0N 2#m];
f:`$":",x:.z.x 0;n:2#-9#x
system"rm -f fifo",n," && mkfifo fifo",n;
system"zcat ",(1_string f)," >fifo",n," &";
td:{` sv`:trips,(`$string x),`trips`}
i:-1;
wr:{td[(`$n),(i+:1),y]set update `p#passenger_count from delete year from select from x where year=y}
rd:.Q.fc[{flip c!(t;",")0:x}];sr:{`year`passenger_count xasc update year:pickup_datetime.year,first each cab_type from x}
\t .Q.fpn[{wr[q]each exec distinct year from q:update `p#year from sr rd x};`$":fifo",n;550000000]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment