merge some files

analysis log file portal

#!/bin/gawk
BEGIN{
	cnt=1;
	cnt_2=0;
	cnt_10=0;
	cnt_499=0;
	cnt_500=0;
	total_time=0;
	cnt_portal=0;
	cnt_portal_2=0;
	cnt_portal_10=0;
	cnt_portal_499=0;
	cnt_portal_500=0;
	portal_total_time=0;
}
{
	cnt++;
	if($10==499){
		cnt_499++;
	}
	if($10>=500&&$10<600){
		cnt_500++;
	}
	match($NF,/\"(.*)\"/,time);
	total_time+=time[1];
	if(time[1]>10){
		cnt_10++;
	}
	if(time[1]>2){
		cnt_2++;
	}
	if($0 ~ /GET \/auth_server\/index.php\/login\/login_in\//){
		cnt_portal++;
		if($10==499){
                	cnt_portal_499++;
        	}
        	if($10>=500&&$10<600){
                	cnt_portal_500++;
        	}
		if(time[1]>10){
         	       cnt_portal_10++;
       		}
        	if(time[1]>2){
                	cnt_portal_2++;
        	}
		portal_total_time+=time[1];
	}

}
END{
	print "cnt,cnt_2,cnt_10,cnt_499,cnt_500"
	print cnt",",cnt_2",",cnt_10",",cnt_499",",cnt_500
	print "\n"
	print "cnt,  total_time/cnt,  cnt_2/cnt,  cnt_10/cnt,  cnt_499/cnt,  cnt_500/cnt"
	print  " " cnt",\n",
		total_time/cnt",\n",
		cnt_2/cnt",\n",
		cnt_10/cnt",\n",
		cnt_499/cnt",\n",
		cnt_500/cnt
}

analysis log file portal login

#!/bin/gawk
BEGIN{
	cnt_portal=0;
	cnt_portal_2=0;
	cnt_portal_10=0;
	cnt_portal_499=0;
	cnt_portal_500=0;
	portal_total_time=0;
}
{
		match($NF,/\"(.*)\"/,time);
		cnt_portal++;
		if($10==499){
                	cnt_portal_499++;
        	}
        	if($10>=500&&$10<600){
                	cnt_portal_500++;
        	}
		if(time[1]>10){
         	       cnt_portal_10++;
       		}
        	if(time[1]>2){
                	cnt_portal_2++;
        	}
		portal_total_time+=time[1];


}
END{
	print "\n"
	print "cnt_portal,  protal_total_time/cnt_portal,  cnt_portal_2/cnt_portal,  cnt_portal_10/cnt_portal,  cnt_portal_499/cnt_portal, cnt_portal_500/cnt_protal"
	print  " "cnt_portal",\n",
		portal_total_time/cnt_portal",\n",
		cnt_portal_2/cnt_portal",\n",
		cnt_portal_10/cnt_portal",\n",
		cnt_portal_499/cnt_portal",\n",
		cnt_portal_500/cnt_portal
}

gistfile1.sh

#!/bin/sh
# author: zane5
# date: 201505
# version: 0.0.0
# this file for merged the portal log file

# echo "It is a test for merge"
 yesterday=`date -d yesterday +%Y%m%d`
 #echo "Prepare process $yesterday data"

 if [ $1 != $yesterday ]; then
	echo $1
	yesterday=$1
	echo "Prepare process $yesterday data"
 else
	echo "Prepare process $yesterday data"
 fi


 path_portal01="/01/"
 path_portal02="/02/"
 path_portal01_yesterday=$path_portal01$yesterday"/"
 echo "checking... $path_portal01_yesterday"
 path_portal02_yesterday=$path_portal02$yesterday"/"
 echo "checking... $path_portal02_yesterday"
 path_portal_merged="/merged/"
 path_cif="/ciiff-log/"
 path_cif_yesterday=$path_cif$yesterday"/"
 echo "checking... $path_cif_yesterday"
 path_cif_merged="/ciiff-merged/"


 if [ -d "$path_cif_merged" ]; then
   `rm -rf $path_cif_merged`
	echo "clear.... $path_cif_merged"
   mkdir $path_cif_merged
 else
  mkdir "$path_cif_merged"

 fi
 if [ -d "$path_portal_merged" ]; then
	`rm -rf $path_portal_merged`
	echo "clear... $path_portal_merged"
	mkdir $path_portal_merged
 else
  mkdir $path_portal_merged
 fi


 for i in $path_portal01_yesterday"access_*"
 do
	cat $i >> $path_portal_merged"yesterday_log_01.log";
	echo  "merge portal01 log"
 done

 for i in $path_portal02_yesterday"access_*"
 do
	cat $i >> $path_portal_merged"yesterday_log_02.log";
	echo "merge portal02 log"
 done

 for i in $path_portal_merged"yesterday_log_0*"
 do
	cat $i >> $path_portal_merged"yesterday_portal_log_merged.log"
	echo "merged all portal files and process the data ..."
 done

 echo "generate the portal login data..."
 grep "login_in" "/merged/yesterday_portal_log_merged.log" > "/media/data2000G10/portal-merged/yesterday_portal_login_in.log"

	`gawk -f /tools/portal.awk /portal-merged/yesterday_portal_log_merged.log > /technical_index/"portal_"$yesterday".log"`
 echo "generate portal_"$yesterday".log"

	`gawk -f /tools/portal_login_in.awk /portal-merged/yesterday_portal_login_in.log > /technical_index/"portal_login_"$yesterday".log"`

 # $path_portal01_yesterday"/access_*" > "merged_portal01"$yesterday".log"

 for i in $path_cif_yesterday"access_*"
 do
	cat $i >> $path_cif_merged"yesterday_cif_merged.log";
	echo "merged all cif files, and process the data ..."
 done

# echo generate $path_cif_merged"yesterday_cif_merged.log";

 `gawk -f /tools/cif.awk $path_cif_merged"yesterday_cif_merged.log" > /technical_index/"cif_"$yesterday".log"`
 echo "generated "cif"$yesterday".log""


 if [ -d "$path_cif_merged" ]; then
   `rm -rf $path_cif_merged`
	echo "clear.... $path_cif_merged"
   mkdir $path_cif_merged
 else
  mkdir "$path_cif_merged"

 fi
 if [ -d "$path_portal_merged" ]; then
	`rm -rf $path_portal_merged`
	echo "clear... $path_portal_merged"
	mkdir $path_portal_merged
 else
  mkdir $path_portal_merged
 fi