Skip to content

Instantly share code, notes, and snippets.

Qi Qi qi-qi

Block or report user

Report or block qi-qi

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
tar czf result.tar.gz -C `pwd` .
curl --data-binary @result.tar.gz
View kinesis-agent
git clone
sudo ./setup --install
Configuration file installed at: /etc/aws-kinesis/agent.json
Configuration details:
"cloudwatch.emitMetrics": true,
"kinesis.endpoint": "",
qi-qi /
Last active Nov 13, 2019
sample code to put record to kinesis firehose
import boto3
import json
client = boto3.client('firehose', aws_access_key_id='aaa', aws_secret_access_key='bbb', region_name='eu-west-1')
payload1 = json.dumps({
'id': 111,
'name': 'QiQi',
'email': ''
View spark-bitset-sample.scala
val ranges = collect_set(struct($"from", $"to")).as("from_to")
.withColumn("bytes_sum_unique", Util.findUniqueBytesUDF($"from_to"))
// Use the BitSet from java.util.BitSet() due to performance
val findUniqueBytesUDF: UserDefinedFunction = udf { ranges: Seq[Row] =>
.map(x => (x.getAs[Int]("legit_from"), x.getAs[Int]("legit_to")))
.aggregate(new java.util.BitSet())((bitset, range) => {
## Colab Notebook:
import requests
from bs4 import BeautifulSoup
from datetime import datetime
keyword = 'ipad pro' #@param {type:"string"}
query = '+'.join(keyword.split(' '))
dt_fmt = '%Y-%m-%d %H:%M:%S'
import requests
import json
from collections import OrderedDict
d1 = {}
d2 = {}
personal_number = "yyyymmdd-xxxx"
for i in range(1000000, 1000400):
View redshit.sql
create table flights
year smallint,
month smallint,
day smallint,
carrier varchar(80) distkey,
origin char(3),
dest char(3),
aircraft_code char(3),
miles int,
set -euxo pipefail
View spark-bitset-udf-collect_list-struct.scala
test("hello-rdd") {
// sample input
val df = Seq(
("a", 0, 200),
("a", 1000, 2000),
("a", 150, 160),
("b", 0, 2),
("b", 2, 8),
("b", 5, 15),
("c", 5, 15),
qi-qi / athena-sql-snippet.sql
Last active Jun 27, 2019
Athena SQL Snippet
View athena-sql-snippet.sql
select dt, count(*), count(distinct batch_id), count(distinct request_id)
from data_extract.requests_batch_parquet
group by dt
order by dt;
select dt, count(*), count(distinct batch_id), count(distinct request_id)
from data_extract.requests_ad_parquet
group by dt
order by dt;
You can’t perform that action at this time.