Skip to content

Instantly share code, notes, and snippets.

View ldgourmetALS.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import itertools
import csv
import datetime
import time
from math import sqrt
from operator import add
@yoshi0309
yoshi0309 / td-agent.conf
Created Oct 27, 2014
td-agent.conf for Solr log for parsing query log.
View td-agent.conf
<source>
type tail
path /opt/solr/solr-4.9.0/example/logs/solr.log
pos_file /var/log/td-agent/solr.log.pos
tag raw.solr.log
format /^(?<loglevel>[^ ]*) (?<hyp>[^ ]*) (?<time>[^ ]* [^ ]*) (?<class>[^ ]*) \[(?<core>[^ ]*)\] webapp=\/(?<webapp>[^ ]*) path=\/(?<path>[^ ]*) params={(?<params>[^ ]*)} hits=(?<hits>[^ ]*) status=(?<status>[^ ]*) QTime=(?<Qtime>[^ ]*) /
time_format %Y-%m-%d %H:%M:%S.%L;
</source>
<match raw.solr.log>
@yoshi0309
yoshi0309 / deleteByQuery.py
Last active Oct 26, 2018
delete documents by query result for Amazon CloudSearch.
View deleteByQuery.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
import urllib
import urllib2
import json
# you need to set your domain endpoints.
SEARCH_ENDPOINT = "XXXXX.us-east-1.cloudsearch.amazonaws.com"
DOCUMENT_ENDPOINT = "XXXXX.us-east-1.cloudsearch.amazonaws.com"
@yoshi0309
yoshi0309 / ExceptionHandler.java
Created Jul 17, 2018
CuratorFramework Experiment handling RuntimeException which throws in a Listener.
View ExceptionHandler.java
import java.util.concurrent.TimeUnit;
public class ExceptionHandler
implements Thread.UncaughtExceptionHandler {
private final long WAIT_TIME = 60L;
@Override public void uncaughtException(Thread thread, Throwable e) {
// -------------------------------------------------
// RuntimeException which was throwed in TestListener should handled here, but not.
@yoshi0309
yoshi0309 / 0_reuse_code.js
Created Nov 14, 2016
Here are some things you can do with Gists in GistBox.
View 0_reuse_code.js
// Use Gists to store code you would like to remember later on
console.log(window); // log the "window" object to the console
@yoshi0309
yoshi0309 / App.java
Created May 19, 2014
Apache Tika 1.5 - AutoDetectParser example
View App.java
package yoshida.tika_sample;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
View bootstrap-urge.sh
#!/bin/bash -ex
# run only on MASTER node.
if grep -Fq "\"isMaster\": true" /mnt/var/lib/info/instance.json ; then
# open the ssh port...
perl -pi -e 's/^#?Port 22$/Port 22\nPort 443/' /etc/ssh/sshd_config
/etc/init.d/sshd restart
# install scala and sbt.
View importCsv2DDB.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import boto
import sys
import csv
"""
Import CSV data to your DynamoDB.
usage : importCSV2DDB.py -f <csv filepath>
@yoshi0309
yoshi0309 / ElasticsearchRecordWriter.java
Created Oct 22, 2014
ElasticsearchRecordWriter.java (using JSON over HTTP not API)
View ElasticsearchRecordWriter.java
package com.yoslab.process.writer;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.util.ArrayList;
View logstash_template_4_solr.json
{
"template" : "logstash-*",
"mappings": {
"solr_log": {
"properties": {
"@log_name": {
"type": "string",
"index" : "not_analyzed"
},
"@timestamp": {
You can’t perform that action at this time.