Added PrivateBin probing.
This commit is contained in:
parent
5d3e05d6f1
commit
50ec907ac8
3 changed files with 381 additions and 0 deletions
|
|
@ -0,0 +1,90 @@
|
|||
/*
|
||||
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDate;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.metrics.TimeMark;
|
||||
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||
|
||||
/**
|
||||
* The Class PrivatebinDataAnalyzer.
|
||||
*/
|
||||
public class PrivatebinDataAnalyzer
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(PrivatebinDataAnalyzer.class);
|
||||
|
||||
/**
|
||||
* Instantiates a new http access log prober.
|
||||
*/
|
||||
private PrivatebinDataAnalyzer()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param httpAccessLogs
|
||||
* the source
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
System.out.println("Probing directory [" + dataDirectory + "]");
|
||||
|
||||
result = new PathCounters();
|
||||
|
||||
if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
|
||||
{
|
||||
LocalDate now = LocalDate.now();
|
||||
String year = TimeMark.yearOf(now).toString();
|
||||
String yearMonth = TimeMark.yearMonthOf(now).toString();
|
||||
String yearWeek = TimeMark.yearWeekOf(now).toString();
|
||||
String date = TimeMark.dayOf(now).toString();
|
||||
|
||||
// metrics.service.files.bytes
|
||||
long size = FileUtils.sizeOfDirectory(dataDirectory);
|
||||
result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
|
||||
|
||||
// metrics.pastebins.count
|
||||
long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
|
||||
result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
else
|
||||
{
|
||||
System.out.println("WARNING: Privatebin data path is not valid.");
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,215 @@
|
|||
/*
|
||||
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.metrics.UserCounters;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
|
||||
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||
import fr.devinsy.statoolinfos.util.LineIterator;
|
||||
|
||||
/**
|
||||
* The Class HttpAccessLogProber.
|
||||
*/
|
||||
public class PrivatebinHttpLogAnalyzer
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(PrivatebinHttpLogAnalyzer.class);
|
||||
|
||||
private PathCounters counters;
|
||||
private UserCounters users;
|
||||
private UserCounters ipv4Users;
|
||||
private UserCounters ipv6Users;
|
||||
|
||||
/**
|
||||
* Instantiates a new http access log prober.
|
||||
*/
|
||||
public PrivatebinHttpLogAnalyzer()
|
||||
{
|
||||
this.counters = new PathCounters();
|
||||
this.users = new UserCounters();
|
||||
this.ipv4Users = new UserCounters();
|
||||
this.ipv6Users = new UserCounters();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the counters.
|
||||
*
|
||||
* @return the counters
|
||||
*/
|
||||
public PathCounters getCounters()
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
result = new PathCounters();
|
||||
result.putAll(this.counters);
|
||||
|
||||
result.putAll(this.users.getCounters("metrics.service.users"));
|
||||
result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
|
||||
result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param file
|
||||
* the file
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public void probe(final File file, final String patternRegex) throws IOException
|
||||
{
|
||||
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
|
||||
|
||||
//
|
||||
Pattern pattern;
|
||||
if (patternRegex == null)
|
||||
{
|
||||
pattern = HttpAccessLogAnalyzer.COMBINED_PATTERN;
|
||||
}
|
||||
else
|
||||
{
|
||||
pattern = Pattern.compile(patternRegex);
|
||||
}
|
||||
|
||||
//
|
||||
LineIterator iterator = new LineIterator(file);
|
||||
while (iterator.hasNext())
|
||||
{
|
||||
String line = iterator.next();
|
||||
|
||||
try
|
||||
{
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, pattern);
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
else
|
||||
{
|
||||
probeLog(log);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe log.
|
||||
*
|
||||
* @param log
|
||||
* the log
|
||||
*/
|
||||
public void probeLog(final HttpAccessLog log)
|
||||
{
|
||||
if (log != null)
|
||||
{
|
||||
// logger.info("LINE IS MATCHING [{}]", log);
|
||||
// logger.info(log.getHttpUserAgent().toString());
|
||||
|
||||
// General HTTP access logs.
|
||||
String year = log.getYear();
|
||||
String yearMonth = log.getYearMonth();
|
||||
String yearWeek = log.getYearWeek();
|
||||
String date = log.getDate();
|
||||
|
||||
// metrics.service.users
|
||||
// metrics.service.users.ipv4
|
||||
// metrics.service.users.ipv6
|
||||
if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
|
||||
{
|
||||
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
|
||||
this.users.put(key, year, yearMonth, yearWeek, date);
|
||||
|
||||
if (log.isIPv4())
|
||||
{
|
||||
this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
|
||||
}
|
||||
}
|
||||
|
||||
// metrics.pastebins.created
|
||||
// metrics.pastebins.read
|
||||
// metrics.pastebins.deleted
|
||||
if (StringUtils.startsWith(log.getRequest(), "POST "))
|
||||
{
|
||||
this.counters.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
|
||||
{
|
||||
if (StringUtils.contains(log.getRequest(), "&deletetoken="))
|
||||
{
|
||||
this.counters.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.counters.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param httpAccessLogs
|
||||
* the source
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public static PathCounters probe(final String httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
PrivatebinHttpLogAnalyzer analyzer = new PrivatebinHttpLogAnalyzer();
|
||||
|
||||
for (File file : FilesUtils.searchByWildcard(httpAccessLogs))
|
||||
{
|
||||
analyzer.probe(file, httpRegex);
|
||||
}
|
||||
|
||||
result = analyzer.getCounters();
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
/*
|
||||
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
|
||||
/**
|
||||
* The Class PrivatebinProber.
|
||||
*/
|
||||
public class PrivatebinProber
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(PrivatebinProber.class);
|
||||
|
||||
/**
|
||||
* Instantiates a new privatebin prober.
|
||||
*/
|
||||
public PrivatebinProber()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param logs
|
||||
* the logs
|
||||
* @param databaseConfig
|
||||
* the database config
|
||||
* @throws StatoolInfosException
|
||||
* @throws IOException
|
||||
*/
|
||||
public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
// metrics.service.users
|
||||
// metrics.service.users.ipv4
|
||||
// metrics.service.users.ipv6
|
||||
// metrics.pastebins.created
|
||||
// metrics.pastebins.read
|
||||
// metrics.pastebins.deleted
|
||||
result = PrivatebinHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
|
||||
|
||||
// metrics.service.files.bytes
|
||||
// metrics.pastebins.count
|
||||
result.putAll(PrivatebinDataAnalyzer.probe(dataPath));
|
||||
|
||||
// metrics.pastebins.purged
|
||||
// purged = count(n-1) - count(n) + created - delete
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue