Skip to content

Instantly share code, notes, and snippets.

@mertyildiran
Last active March 24, 2023 01:36
Show Gist options
  • Save mertyildiran/9bcc600f0867929765c7451334ffcfca to your computer and use it in GitHub Desktop.
Save mertyildiran/9bcc600f0867929765c7451334ffcfca to your computer and use it in GitHub Desktop.
JavaScript code with issues
var kflPcapS3Data = null;
wrapper.kflPcapS32 = function (data, params) {
function kflPcapS3detect(data) {
if (kflPcapS3Data === null)
return;
kflPcapS3Data.kflArr.forEach(function (kflQuery, idx) {
if (kfl.match(kflQuery, data)) {
kflPcapS3Data.pcapInfoArr[idx].pcapArr.push(data.stream);
if (kflPcapS3Data.verbose)
console.log("KFL/PCAP MATCH: KFL=" + kflQuery + "; PCAP=" + data.stream + "; Idx=" + idx + "; files=" + kflPcapS3Data.pcapInfoArr[idx].pcapArr.length + "; time: " + kflPcapS3Data.pcapInfoArr[idx].time);
}
});
}
function kflPcapS3Job() {
console.log(Date().toLocaleString() + ":kflPcapS3Job");
var now = Date.now();
if (kflPcapS3Data.jobTimePeriod === undefined || now > kflPcapS3Data.jobTime + kflPcapS3Data.jobTimePeriod) {
kflPcapS3Data.jobTime = now;
kflPcapS3Data.pcapInfoArr.forEach(function (pcapInfo, idx) {
if (
(
kflPcapS3Data.maxL4Streams && (kflPcapS3Data.pcapInfoArr[idx].pcapArr.length > kflPcapS3Data.maxL4Streams)
) ||
(
(now >= kflPcapS3Data.pcapInfoArr[idx].time + kflPcapS3Data.maxMinutesInMS) &&
(kflPcapS3Data.pcapInfoArr[idx].pcapArr.length > 0)
)
) {
kflPcapS3Data.pcapInfoArr[idx].time = now;
kflPcapS3upload(idx);
}
});
}
if (
(kflPcapS3Data.logUploadTimePeriod === undefined || now > kflPcapS3Data.logUploadTime + kflPcapS3Data.logUploadTimePeriod) &&
kflPcapS3Data.progressLog.length
) {
kflPcapS3Data.logUploadTime = now;
kflPcapS3JobLog();
}
}
function kflPcapS3JobLog() {
console.log(Date().toLocaleString() + ":kflPcapS3JobLog");
file.write(kflPcapS3Data.progressLogFile, JSON.stringify(kflPcapS3Data.progressLog));
if (kflPcapS3Data.verbose) console.log("kflPcapS3jobLog|logFile: ", kflPcapS3Data.progressLogFile);
var s3Time = Date.now();
var location = vendor.s3.put(
kflPcapS3Data.awsRegion,
kflPcapS3Data.awsAccessKeyId,
kflPcapS3Data.awsSecretAccessKey,
kflPcapS3Data.s3Bucket,
kflPcapS3Data.progressLogFile
);
s3Time = Date.now() - s3Time;
var msg = "Updated Progress Log: " + location + "; S3 upload time: " + s3Time + "ms";
if (kflPcapS3Data.slackWebhook)
vendor.slack(
kflPcapS3Data.slackWebhook,
"Notification", msg,
"#ff0000"
);
if (kflPcapS3Data.slackAuthToken && kflPcapS3Data.slackChannelId)
vendor.slackBot(
kflPcapS3Data.slackAuthToken,
kflPcapS3Data.slackChannelId,
"Notification (kflPcapS3)",
msg,
"#ff0000"
);
console.log(Date().toLocaleString() + ":" + msg);
}
function kflPcapS3upload(idx) {
try {
var newTempDir = file.mkdirTemp("pcaps3idx" + idx, "");
var pcapFilesS3 = kflPcapS3Data.pcapInfoArr[idx].pcapArr;
kflPcapS3Data.pcapInfoArr[idx].pcapArr = [];
if (kflPcapS3Data.verbose)
console.log("pcap.snapshot: " + pcapFilesS3.length + " files");
var snapshotTime = Date.now();
var pcapFile = pcap.snapshot(pcapFilesS3);
snapshotTime = Date.now() - snapshotTime;
if (kflPcapS3Data.verbose) console.log("pcapFile: ", pcapFile);
file.move(pcapFile, newTempDir);
var nameResolutionHistory = pcap.nameResolutionHistory();
file.write(newTempDir + "/name_resolution_history.json", JSON.stringify(nameResolutionHistory));
file.write(
newTempDir + "/content.json",
JSON.stringify(
{
pcap_file_name: pcapFile,
time: Date().toLocaleString(),
kfl_index: idx,
kfl_query: kflPcapS3Data.pcapInfoArr[idx].kfl,
l4_streams: pcapFilesS3
}
)
);
var tarFile = file.tar(newTempDir);
var newTarFile = "kfl_" + idx + "_" + tarFile;
file.move(tarFile, newTarFile);
if (kflPcapS3Data.verbose) console.log("pcapS3Job|tarFile: ", newTarFile);
var s3Time = Date.now();
var location = vendor.s3.put(
kflPcapS3Data.awsRegion,
kflPcapS3Data.awsAccessKeyId,
kflPcapS3Data.awsSecretAccessKey,
kflPcapS3Data.s3Bucket,
newTarFile
);
s3Time = Date.now() - s3Time;
file.delete(newTempDir);
file.delete(newTarFile);
var msg = "New PCAP: " + location + "; L4 streams: " + pcapFilesS3.length + "; KFL: \"" + kflPcapS3Data.pcapInfoArr[idx].kfl + "\"; Snapshot time: " + snapshotTime + "ms; S3 upload time: " + s3Time + "ms";
if (kflPcapS3Data.slackWebhook)
vendor.slack(
kflPcapS3Data.slackWebhook,
"Notification", msg,
"#ff0000"
);
if (kflPcapS3Data.slackAuthToken && kflPcapS3Data.slackChannelId)
vendor.slackBot(
kflPcapS3Data.slackAuthToken,
kflPcapS3Data.slackChannelId,
"Notification (kflPcapS3)",
msg,
"#ff0000"
);
console.log(Date().toLocaleString() + ":" + msg);
kflPcapS3Data.progressLog.push({
file: newTarFile,
s3_url: location,
time: Date().toLocaleString(),
kfl_index: idx,
kfl_query: kflPcapS3Data.pcapInfoArr[idx].kfl,
});
} catch (err) {
console.error(err);
}
}
kflPcapS3detect(data);
if (kflPcapS3Data !== null)
return;
if (!data || (typeof params !== 'object') || !params) {
console.error("kflPcapS3: Expected data and params. Got: ", JSON.stringify({
data: data,
params: params
}));
return;
}
kflPcapS3Data = { // set defaults
kflArr: [], // Mandatory
/* the rest of the properties are optional */
verbose: false,
slackWebhook: null,
slackAuthToken: null,
slackAuthChannelId: null,
maxMinutes: 60,
maxL4Streams: 100000,
awsRegion: env.AWS_REGION,
awsAccessKeyId: env.AWS_ACCESS_KEY_ID,
awsSecretAccessKey: env.AWS_SECRET_ACCESS_KEY,
s3Bucket: env.S3_BUCKET,
pcapInfoArr: [],
firstTime: true,
maxMinutesInMS: 3600000,
progressLogFile: file.temp("kflPcapS3_log_", "", "json"),
progressLog: [],
logUploadTime: Date.now(),
jobTime: Date.now(),
logUploadTimePeriod: 3600000
}
if (params.kflArr !== undefined)
kflPcapS3Data.kflArr = params.kflArr;
else {
console.error("kflPcapS3: kflArr is mandatory. Got: ", JSON.stringify(params));
return;
}
if (params.awsRegion !== undefined)
kflPcapS3Data.awsRegion = params.awsRegion;
if (params.awsAccessKeyId !== undefined)
kflPcapS3Data.awsAccessKeyId = params.awsAccessKeyId;
if (params.awsAccessKeyId !== undefined)
kflPcapS3Data.awsSecretAccessKey = params.awsSecretAccessKey;
if (params.s3Bucket !== undefined)
kflPcapS3Data.s3Bucket = params.s3Bucket;
if ((kflPcapS3Data.s3Bucket === undefined) || (kflPcapS3Data.awsSecretAccessKey === undefined) || (kflPcapS3Data.awsRegion === undefined)) {
console.error("kflPcapS3: One or more of AWS peoprties is missing.");
return;
}
if (params.clear === true)
vendor.s3.clear(
kflPcapS3Data.awsRegion,
kflPcapS3Data.awsAccessKeyId,
kflPcapS3Data.awsSecretAccessKey,
kflPcapS3Data.s3Bucket
);
if (params.verbose !== undefined)
kflPcapS3Data.verbose = params.verbose;
kflPcapS3Data.slackWebhook = params.slackWebhook;
kflPcapS3Data.slackAuthToken = params.slackAuthToken;
kflPcapS3Data.slackAuthChannelId = params.slackAuthChannelId;
if (params.maxMinutes !== undefined)
kflPcapS3Data.maxMinutes = params.maxMinutes;
if (params.maxL4Streams !== undefined)
kflPcapS3Data.maxL4Streams = params.maxL4Streams;
kflPcapS3Data.maxMinutesInMS = kflPcapS3Data.maxMinutes * 60000;
kflPcapS3Data.kflArr.forEach(function (kflQuery, idx) {
kflPcapS3Data.pcapInfoArr[idx] = {
pcapArr: [],
kfl: kflQuery,
time: Date.now()
};
});
jobs.schedule("kfl-pcap-s3", "*/30 * * * * *", kflPcapS3Job);
}
var KFL_PCAP_S3_KFL_ARR = [
"http and (response.status==500)",
"dns",
];
function onItemCaptured(data) {
wrapper.kflPcapS32(data, {
kflArr: KFL_PCAP_S3_KFL_ARR, // Mandatory
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment