Added PrivateBin probing.
This commit is contained in:
parent
5d3e05d6f1
commit
50ec907ac8
3 changed files with 381 additions and 0 deletions
|
|
@ -0,0 +1,90 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of StatoolInfos, simple service statistics tool.
|
||||||
|
*
|
||||||
|
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* StatoolInfos is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.time.LocalDate;
|
||||||
|
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||||
|
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||||
|
import fr.devinsy.statoolinfos.metrics.TimeMark;
|
||||||
|
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class PrivatebinDataAnalyzer.
|
||||||
|
*/
|
||||||
|
public class PrivatebinDataAnalyzer
|
||||||
|
{
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(PrivatebinDataAnalyzer.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new http access log prober.
|
||||||
|
*/
|
||||||
|
private PrivatebinDataAnalyzer()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Probe.
|
||||||
|
*
|
||||||
|
* @param httpAccessLogs
|
||||||
|
* the source
|
||||||
|
* @throws IOException
|
||||||
|
* Signals that an I/O exception has occurred.
|
||||||
|
* @throws StatoolInfosException
|
||||||
|
* the statool infos exception
|
||||||
|
*/
|
||||||
|
public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
|
||||||
|
{
|
||||||
|
PathCounters result;
|
||||||
|
|
||||||
|
System.out.println("Probing directory [" + dataDirectory + "]");
|
||||||
|
|
||||||
|
result = new PathCounters();
|
||||||
|
|
||||||
|
if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
|
||||||
|
{
|
||||||
|
LocalDate now = LocalDate.now();
|
||||||
|
String year = TimeMark.yearOf(now).toString();
|
||||||
|
String yearMonth = TimeMark.yearMonthOf(now).toString();
|
||||||
|
String yearWeek = TimeMark.yearWeekOf(now).toString();
|
||||||
|
String date = TimeMark.dayOf(now).toString();
|
||||||
|
|
||||||
|
// metrics.service.files.bytes
|
||||||
|
long size = FileUtils.sizeOfDirectory(dataDirectory);
|
||||||
|
result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
|
||||||
|
|
||||||
|
// metrics.pastebins.count
|
||||||
|
long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
|
||||||
|
result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
System.out.println("WARNING: Privatebin data path is not valid.");
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,215 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of StatoolInfos, simple service statistics tool.
|
||||||
|
*
|
||||||
|
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* StatoolInfos is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||||
|
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||||
|
import fr.devinsy.statoolinfos.metrics.UserCounters;
|
||||||
|
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
|
||||||
|
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
|
||||||
|
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||||
|
import fr.devinsy.statoolinfos.util.LineIterator;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class HttpAccessLogProber.
|
||||||
|
*/
|
||||||
|
public class PrivatebinHttpLogAnalyzer
|
||||||
|
{
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(PrivatebinHttpLogAnalyzer.class);
|
||||||
|
|
||||||
|
private PathCounters counters;
|
||||||
|
private UserCounters users;
|
||||||
|
private UserCounters ipv4Users;
|
||||||
|
private UserCounters ipv6Users;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new http access log prober.
|
||||||
|
*/
|
||||||
|
public PrivatebinHttpLogAnalyzer()
|
||||||
|
{
|
||||||
|
this.counters = new PathCounters();
|
||||||
|
this.users = new UserCounters();
|
||||||
|
this.ipv4Users = new UserCounters();
|
||||||
|
this.ipv6Users = new UserCounters();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the counters.
|
||||||
|
*
|
||||||
|
* @return the counters
|
||||||
|
*/
|
||||||
|
public PathCounters getCounters()
|
||||||
|
{
|
||||||
|
PathCounters result;
|
||||||
|
|
||||||
|
result = new PathCounters();
|
||||||
|
result.putAll(this.counters);
|
||||||
|
|
||||||
|
result.putAll(this.users.getCounters("metrics.service.users"));
|
||||||
|
result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
|
||||||
|
result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Probe.
|
||||||
|
*
|
||||||
|
* @param file
|
||||||
|
* the file
|
||||||
|
* @throws IOException
|
||||||
|
* Signals that an I/O exception has occurred.
|
||||||
|
* @throws StatoolInfosException
|
||||||
|
* the statool infos exception
|
||||||
|
*/
|
||||||
|
public void probe(final File file, final String patternRegex) throws IOException
|
||||||
|
{
|
||||||
|
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
|
||||||
|
|
||||||
|
//
|
||||||
|
Pattern pattern;
|
||||||
|
if (patternRegex == null)
|
||||||
|
{
|
||||||
|
pattern = HttpAccessLogAnalyzer.COMBINED_PATTERN;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
pattern = Pattern.compile(patternRegex);
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
LineIterator iterator = new LineIterator(file);
|
||||||
|
while (iterator.hasNext())
|
||||||
|
{
|
||||||
|
String line = iterator.next();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, pattern);
|
||||||
|
if (log == null)
|
||||||
|
{
|
||||||
|
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
probeLog(log);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
|
||||||
|
exception.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Probe log.
|
||||||
|
*
|
||||||
|
* @param log
|
||||||
|
* the log
|
||||||
|
*/
|
||||||
|
public void probeLog(final HttpAccessLog log)
|
||||||
|
{
|
||||||
|
if (log != null)
|
||||||
|
{
|
||||||
|
// logger.info("LINE IS MATCHING [{}]", log);
|
||||||
|
// logger.info(log.getHttpUserAgent().toString());
|
||||||
|
|
||||||
|
// General HTTP access logs.
|
||||||
|
String year = log.getYear();
|
||||||
|
String yearMonth = log.getYearMonth();
|
||||||
|
String yearWeek = log.getYearWeek();
|
||||||
|
String date = log.getDate();
|
||||||
|
|
||||||
|
// metrics.service.users
|
||||||
|
// metrics.service.users.ipv4
|
||||||
|
// metrics.service.users.ipv6
|
||||||
|
if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
|
||||||
|
{
|
||||||
|
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
|
||||||
|
this.users.put(key, year, yearMonth, yearWeek, date);
|
||||||
|
|
||||||
|
if (log.isIPv4())
|
||||||
|
{
|
||||||
|
this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// metrics.pastebins.created
|
||||||
|
// metrics.pastebins.read
|
||||||
|
// metrics.pastebins.deleted
|
||||||
|
if (StringUtils.startsWith(log.getRequest(), "POST "))
|
||||||
|
{
|
||||||
|
this.counters.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
|
||||||
|
}
|
||||||
|
else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
|
||||||
|
{
|
||||||
|
if (StringUtils.contains(log.getRequest(), "&deletetoken="))
|
||||||
|
{
|
||||||
|
this.counters.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
this.counters.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Probe.
|
||||||
|
*
|
||||||
|
* @param httpAccessLogs
|
||||||
|
* the source
|
||||||
|
* @throws IOException
|
||||||
|
* Signals that an I/O exception has occurred.
|
||||||
|
* @throws StatoolInfosException
|
||||||
|
* the statool infos exception
|
||||||
|
*/
|
||||||
|
public static PathCounters probe(final String httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
|
||||||
|
{
|
||||||
|
PathCounters result;
|
||||||
|
|
||||||
|
PrivatebinHttpLogAnalyzer analyzer = new PrivatebinHttpLogAnalyzer();
|
||||||
|
|
||||||
|
for (File file : FilesUtils.searchByWildcard(httpAccessLogs))
|
||||||
|
{
|
||||||
|
analyzer.probe(file, httpRegex);
|
||||||
|
}
|
||||||
|
|
||||||
|
result = analyzer.getCounters();
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,76 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of StatoolInfos, simple service statistics tool.
|
||||||
|
*
|
||||||
|
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* StatoolInfos is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||||
|
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class PrivatebinProber.
|
||||||
|
*/
|
||||||
|
public class PrivatebinProber
|
||||||
|
{
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(PrivatebinProber.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new privatebin prober.
|
||||||
|
*/
|
||||||
|
public PrivatebinProber()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Probe.
|
||||||
|
*
|
||||||
|
* @param logs
|
||||||
|
* the logs
|
||||||
|
* @param databaseConfig
|
||||||
|
* the database config
|
||||||
|
* @throws StatoolInfosException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
|
||||||
|
{
|
||||||
|
PathCounters result;
|
||||||
|
|
||||||
|
// metrics.service.users
|
||||||
|
// metrics.service.users.ipv4
|
||||||
|
// metrics.service.users.ipv6
|
||||||
|
// metrics.pastebins.created
|
||||||
|
// metrics.pastebins.read
|
||||||
|
// metrics.pastebins.deleted
|
||||||
|
result = PrivatebinHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
|
||||||
|
|
||||||
|
// metrics.service.files.bytes
|
||||||
|
// metrics.pastebins.count
|
||||||
|
result.putAll(PrivatebinDataAnalyzer.probe(dataPath));
|
||||||
|
|
||||||
|
// metrics.pastebins.purged
|
||||||
|
// purged = count(n-1) - count(n) + created - delete
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue