haoch

--- /usr/local/lib/python3.7/site-packages/airflow/models/dagrun.py	2020-08-05 21:09:34.304289571 +0000
+++ /usr/local/lib/python3.7/site-packages/airflow/models/dagrun.py.backup	2020-08-05 07:16:06.909641726 +0000
@@ -21,7 +21,7 @@
 import six
 from sqlalchemy import (
     Column, Integer, String, Boolean, PickleType, Index, UniqueConstraint, func, DateTime, or_,
-    and_, desc
+    and_
 )
 from sqlalchemy.ext.declarative import declared_attr

haoch

{
    "template": "eagle_metric*",
    "settings": {
        "index.cache.field.type": "soft",
        "index.store.compress.stored": true,
        "index.refresh_interval": "5s",
        "index.query.default_field": "name",
        "number_of_shards": 7,
        "number_of_replicas": 2
    },

haoch

    @Override
    public synchronized void status(final PipelineExecutionEntity entity) throws Exception {
        String stormUIUrl= (String) entity.getPipeline().getCluster().getProperties().get(STORM_URL);
        entity.requireUpdate(true);
        if (LogStormConstants.DeployMode.LOCAL.equals(entity.getPipeline().getMode())) {
            if(!ExecutionManager.getInstance().contains(entity.getName())){
                LOG.info("Pipeline instance '{}' is not ready yet",entity.getName());
            }
            PipelineExecutionStatus currentStatus = entity.getStatus();
            PipelineExecutionStatus newStatus = ExecutionManager.getWorkerStatus(ExecutionManager.getInstance().get(entity.getName()).getState());

haoch

input {
	kafka {
  	  zk_connect => "localhost:2181"
  	  topic_id => "input_log_topic"
    }
}

filter{
	grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }

haoch

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *

haoch

# Stream Config
CONFIG {
    "config.resource" -> "application.conf"
    "default.flow.engine" -> "siddhi"
}
 
FLOW {
    # Stream Definition
    define stream metricStream_1 (metricType: string,site: string,host: string,component: string,value: double);
    define stream metricStream_2 (metricType: string,site: string,host: string,component: string,value: double);

haoch

MyApp extends StreamApp{
	/**
	 * Scheme Definition
	 */
	stream("hdfsAuditLogStream",
		datasource = "HDFS",
		attributes = {
			"path" -> string("Path","HDFS Path"),
			"size" -> long("Size","HDFS Path Size")
		}

haoch

source 'https://rubygems.org'

gem 'rufus-scheduler'
gem 'daemons'

haoch

PS1='\[\033[0;32m\]\u\[\033[00m\] in \[\033[0;32m\]$( pwd ) ($( OUT=$( ls -A | wc -l ); echo $OUT ) entries, $(( $( ls -A | wc -l ) - $( ls | wc -l ) )) hidden)\n\$\[\033[;m\] '

haoch

require 'java'
 
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.conf.Configuration
 
jruby_class_loader = JRuby.runtime.getJRubyClassLoader
Java::java.lang.Thread.currentThread.setContextClassLoader(jruby_class_loader)
 
#Parameters
#conf_dir =  ENV['HADOOP_CONF_DIR'] #path to HADOOP_CONF_DIR