Skip to content

Instantly share code, notes, and snippets.

Avatar

Fernando fndiaz

  • São Paulo
View GitHub Profile
View mongo_lag.py
import pymongo
uri = "mongodb://user:pass@localhost:27017/admin"
conn = pymongo.MongoClient(uri)
db = conn['admin']
db_stats = db.command({'replSetGetStatus' :1})
server_secondary = 'name_server:27017'
primary_optime = 0
secondary_optime = 0
View kafka-migrate-topics.py
import kafka
import os
brokers="kb0:9092,kb1:9092,kb2:9092"
zookp="zk0:2181,zk1:2181,zk2:2181"
client = kafka.KafkaClient(brokers)
consumer = kafka.KafkaConsumer(group_id='migrate-topics', bootstrap_servers=brokers.split(","))
topics=consumer.topics()
View debezium.py
#!/usr/bin/python
import requests, re, random, json
import optparse
import time
def get_config(headers, conn, connectors):
path = 'http://localhost:8083/connectors/%s' %(conn)
r = requests.get(path, headers=headers)
config = r.json()
names = name(conn)
View lambda-elblogs-to-elastic.py
import boto3
import re
import certifi
from datetime import datetime
from elasticsearch import Elasticsearch
from elasticsearch import helpers
from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
from elasticsearch import Elasticsearch, RequestsHttpConnection
View police-cwlogs-elastic_post
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
View firehose-agent.json
{
"cloudwatch.endpoint":"monitoring.us-east-1.amazonaws.com",
"cloudwatch.emitMetrics":true,
"firehose.endpoint":"firehose.us-east-1.amazonaws.com",
"flows":[
{
"filePattern":"/var/log/nginx/access.log",
"deliveryStream":"NAME_STREAM",
"dataProcessingOptions":[
{
@fndiaz
fndiaz / connector.py
Created Sep 27, 2018
Connector Firehose Python
View connector.py
import boto3
client = boto3.client('firehose', region_name='us-east-1')
response = client.put_record(
DeliveryStreamName='FIREHOSE_NAME',
Record={
'Data': '{"datetime": "2018-09-09T18:51:27", "msg": "testepython", "status": "OK"}'
}
)
View ratelimit-lambda.js
'use strict';
var http = require('http');
// Load the SDK for JavaScript
const https = require('https');
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
@fndiaz
fndiaz / s3-to-es-lamba.py
Created Aug 15, 2018 — forked from s-fujimoto/s3-to-es-lamba.py
Import Elasticsearch from ELB access log for AWS Lambda Function
View s3-to-es-lamba.py
##################################################
### Elasticsearch host name
ES_HOST = "search-******************.ap-northeast-1.es.amazonaws.com"
### Elasticsearch prefix for index name
INDEX_PREFIX = "awslogs"
#################################################
### ELB access log format keys
ELB_KEYS = ["timestamp", "elb", "client_ip", "client_port", "backend_ip", "backend_port", "request_processing_time", "backend_processing_time", "response_processing_time", "elb_status_code", "backend_status_code", "received_bytes", "sent_bytes", "request_method", "request_url", "request_version", "user_agent"]
@fndiaz
fndiaz / curl.md
Created Jul 13, 2018 — forked from subfuzion/curl.md
curl POST examples
View curl.md

Common Options

-#, --progress-bar Make curl display a simple progress bar instead of the more informational standard meter.

-b, --cookie <name=data> Supply cookie with request. If no =, then specifies the cookie file to use (see -c).

-c, --cookie-jar <file name> File to save response cookies to.

You can’t perform that action at this time.