Skip to content

Instantly share code, notes, and snippets.

View fndiaz's full-sized avatar
:octocat:

Fernando fndiaz

:octocat:
  • São Paulo
View GitHub Profile
import pymongo
uri = "mongodb://user:pass@localhost:27017/admin"
conn = pymongo.MongoClient(uri)
db = conn['admin']
db_stats = db.command({'replSetGetStatus' :1})
server_secondary = 'name_server:27017'
primary_optime = 0
secondary_optime = 0
import kafka
import os
brokers="kb0:9092,kb1:9092,kb2:9092"
zookp="zk0:2181,zk1:2181,zk2:2181"
client = kafka.KafkaClient(brokers)
consumer = kafka.KafkaConsumer(group_id='migrate-topics', bootstrap_servers=brokers.split(","))
topics=consumer.topics()
#!/usr/bin/python
import requests, re, random, json
import optparse
import time
def get_config(headers, conn, connectors):
path = 'http://localhost:8083/connectors/%s' %(conn)
r = requests.get(path, headers=headers)
config = r.json()
names = name(conn)
import boto3
import re
import certifi
from datetime import datetime
from elasticsearch import Elasticsearch
from elasticsearch import helpers
from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
from elasticsearch import Elasticsearch, RequestsHttpConnection
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
{
"cloudwatch.endpoint":"monitoring.us-east-1.amazonaws.com",
"cloudwatch.emitMetrics":true,
"firehose.endpoint":"firehose.us-east-1.amazonaws.com",
"flows":[
{
"filePattern":"/var/log/nginx/access.log",
"deliveryStream":"NAME_STREAM",
"dataProcessingOptions":[
{
@fndiaz
fndiaz / connector.py
Created September 27, 2018 20:27
Connector Firehose Python
import boto3
client = boto3.client('firehose', region_name='us-east-1')
response = client.put_record(
DeliveryStreamName='FIREHOSE_NAME',
Record={
'Data': '{"datetime": "2018-09-09T18:51:27", "msg": "testepython", "status": "OK"}'
}
)
@fndiaz
fndiaz / connector_firehose.php
Last active September 27, 2018 20:24
Conector php for Firehose Stream AWS
<?php
use Aws\Firehose\FirehoseClient;
// Install deps with `composer install`
require 'vendor/autoload.php';
$firehose = new FirehoseClient([
'version' => 'latest',
'region' => 'us-east-1',
'credentials' => [ // OR set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars
'key' => 'key',
'use strict';
var http = require('http');
// Load the SDK for JavaScript
const https = require('https');
var AWS = require('aws-sdk');
// Set the region
AWS.config.update({region: 'us-east-1'});
@fndiaz
fndiaz / s3-to-es-lamba.py
Created August 15, 2018 21:46 — forked from s-fujimoto/s3-to-es-lamba.py
Import Elasticsearch from ELB access log for AWS Lambda Function
##################################################
### Elasticsearch host name
ES_HOST = "search-******************.ap-northeast-1.es.amazonaws.com"
### Elasticsearch prefix for index name
INDEX_PREFIX = "awslogs"
#################################################
### ELB access log format keys
ELB_KEYS = ["timestamp", "elb", "client_ip", "client_port", "backend_ip", "backend_port", "request_processing_time", "backend_processing_time", "response_processing_time", "elb_status_code", "backend_status_code", "received_bytes", "sent_bytes", "request_method", "request_url", "request_version", "user_agent"]