Skip to content

Instantly share code, notes, and snippets.

Qi Qi qi-qi

Block or report user

Report or block qi-qi

Hide content and notifications from this user.

Learn more about blocking users

Contact Support about this user’s behavior.

Learn more about reporting abuse

Report abuse
View GitHub Profile
View misc.sh
tar czf result.tar.gz -C `pwd` .
curl https://bashupload.com/result.tar.gz --data-binary @result.tar.gz
View kinesis-agent
git clone https://github.com/awslabs/amazon-kinesis-agent.git
sudo ./setup --install
...
Configuration file installed at: /etc/aws-kinesis/agent.json
Configuration details:
{
"cloudwatch.emitMetrics": true,
"kinesis.endpoint": "",
@qi-qi
qi-qi / kinesis-firehose-sample.py
Last active Nov 13, 2019
sample code to put record to kinesis firehose
View kinesis-firehose-sample.py
import boto3
import json
client = boto3.client('firehose', aws_access_key_id='aaa', aws_secret_access_key='bbb', region_name='eu-west-1')
payload1 = json.dumps({
'id': 111,
'name': 'QiQi',
'email': 'test@test.com'
})
View spark-bitset-sample.scala
val ranges = collect_set(struct($"from", $"to")).as("from_to")
df.groupBy($"id")
.agg(ranges)
.withColumn("bytes_sum_unique", Util.findUniqueBytesUDF($"from_to"))
// Use the BitSet from java.util.BitSet() due to performance
val findUniqueBytesUDF: UserDefinedFunction = udf { ranges: Seq[Row] =>
ranges
.map(x => (x.getAs[Int]("legit_from"), x.getAs[Int]("legit_to")))
.aggregate(new java.util.BitSet())((bitset, range) => {
View blocket.py
## Colab Notebook: https://colab.research.google.com/drive/1bzT9XYTymi5E-x4C_-tmpla9_Ha2PgQd
import requests
from bs4 import BeautifulSoup
from datetime import datetime
keyword = 'ipad pro' #@param {type:"string"}
query = '+'.join(keyword.split(' '))
dt_fmt = '%Y-%m-%d %H:%M:%S'
View car.py
import requests
import json
from collections import OrderedDict
d1 = {}
d2 = {}
personal_number = "yyyymmdd-xxxx"
for i in range(1000000, 1000400):
View redshit.sql
create table flights
(
year smallint,
month smallint,
day smallint,
carrier varchar(80) distkey,
origin char(3),
dest char(3),
aircraft_code char(3),
miles int,
View sample.sh
#!/bin/bash
set -euxo pipefail
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail
View spark-bitset-udf-collect_list-struct.scala
test("hello-rdd") {
// sample input
val df = Seq(
("a", 0, 200),
("a", 1000, 2000),
("a", 150, 160),
("b", 0, 2),
("b", 2, 8),
("b", 5, 15),
("c", 5, 15),
@qi-qi
qi-qi / athena-sql-snippet.sql
Last active Jun 27, 2019
Athena SQL Snippet
View athena-sql-snippet.sql
select dt, count(*), count(distinct batch_id), count(distinct request_id)
from data_extract.requests_batch_parquet
group by dt
order by dt;
select dt, count(*), count(distinct batch_id), count(distinct request_id)
from data_extract.requests_ad_parquet
group by dt
order by dt;
You can’t perform that action at this time.