Added Privatebin patch log probing.
This commit is contained in:
parent
058b902fbd
commit
950d06fae7
8 changed files with 554 additions and 9 deletions
|
|
@ -158,7 +158,7 @@ conf.probe.gitea.database.password=
|
|||
conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
|
||||
```
|
||||
|
||||
### LibreQR metrics (coming soon)
|
||||
### LibreQR metrics
|
||||
|
||||
Configuration template:
|
||||
|
||||
|
|
@ -211,7 +211,7 @@ conf.probe.httpaccesslog.pattern=
|
|||
conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
|
||||
```
|
||||
|
||||
### PrivateBin metrics (experimental)
|
||||
### PrivateBin metrics (partial)
|
||||
|
||||
Warning: works fine if database, image and comment options are disabled.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* The Enum PrivatebinLogAction.
|
||||
*/
|
||||
public enum PrivatebinLogAction
|
||||
{
|
||||
COMMENT,
|
||||
CREATE,
|
||||
DELETE,
|
||||
READ;
|
||||
|
||||
/**
|
||||
* Of.
|
||||
*
|
||||
* @param value
|
||||
* the value
|
||||
* @return the privatebin log action
|
||||
*/
|
||||
public static PrivatebinLogAction of(final String value)
|
||||
{
|
||||
PrivatebinLogAction result;
|
||||
|
||||
if (StringUtils.equals(value, "COMMENT"))
|
||||
{
|
||||
result = COMMENT;
|
||||
}
|
||||
else if (StringUtils.equals(value, "CREATE"))
|
||||
{
|
||||
result = CREATE;
|
||||
}
|
||||
else if (StringUtils.equals(value, "DELETE"))
|
||||
{
|
||||
result = DELETE;
|
||||
}
|
||||
else if (StringUtils.equals(value, "READ"))
|
||||
{
|
||||
result = READ;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
/*
|
||||
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.metrics.TimeMarkUtils;
|
||||
|
||||
/**
|
||||
* The Class PrivatebinPatchLog.
|
||||
*/
|
||||
public class PrivatebinPatchLog
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(PrivatebinPatchLog.class);
|
||||
|
||||
private LocalDateTime time;
|
||||
private PrivatebinLogAction action;
|
||||
private String id;
|
||||
|
||||
/**
|
||||
* Instantiates a new privatebin patch log.
|
||||
*/
|
||||
public PrivatebinPatchLog()
|
||||
{
|
||||
this.time = null;
|
||||
this.action = null;
|
||||
this.id = null;
|
||||
}
|
||||
|
||||
public PrivatebinLogAction getAction()
|
||||
{
|
||||
return this.action;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the date.
|
||||
*
|
||||
* @return the date
|
||||
*/
|
||||
public String getDate()
|
||||
{
|
||||
String result;
|
||||
|
||||
result = this.time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
public String getId()
|
||||
{
|
||||
return this.id;
|
||||
}
|
||||
|
||||
public LocalDateTime getTime()
|
||||
{
|
||||
return this.time;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year.
|
||||
*
|
||||
* @return the year
|
||||
*/
|
||||
public String getYear()
|
||||
{
|
||||
String result;
|
||||
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearOf(this.time);
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year month.
|
||||
*
|
||||
* @return the year month
|
||||
*/
|
||||
public String getYearMonth()
|
||||
{
|
||||
String result;
|
||||
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearMonthOf(this.time);
|
||||
}
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year week.
|
||||
*
|
||||
* @return the year week
|
||||
*/
|
||||
public String getYearWeek()
|
||||
{
|
||||
String result;
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearWeekOf(this.time);
|
||||
}
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
public void setAction(final PrivatebinLogAction action)
|
||||
{
|
||||
this.action = action;
|
||||
}
|
||||
|
||||
public void setId(final String id)
|
||||
{
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public void setTime(final LocalDateTime time)
|
||||
{
|
||||
this.time = time;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.lang.Object#toString()
|
||||
*/
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
String result;
|
||||
|
||||
result = String.format("[time=%s, action=%s, id=%s]", this.time, this.action, this.id);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,182 @@
|
|||
/*
|
||||
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.privatebin;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.util.LineIterator;
|
||||
|
||||
/**
|
||||
* The Class PrivatebinPatchLogAnalyzer.
|
||||
*/
|
||||
public class PrivatebinPatchLogProber
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(PrivatebinPatchLogProber.class);
|
||||
|
||||
// 2021-11-17 01:51:59 DELETE 7922dd508e0cf9b2
|
||||
public static final Pattern PATCH_LOG_PATTERN = Pattern.compile(
|
||||
"^(?<datetime>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+(?<action>\\S+)\\s+(?<id>\\S+)$");
|
||||
|
||||
/**
|
||||
* Instantiates a new privatebin patch log analyzer.
|
||||
*/
|
||||
private PrivatebinPatchLogProber()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the http log
|
||||
*/
|
||||
public static PrivatebinPatchLog parseLog(final String line)
|
||||
{
|
||||
PrivatebinPatchLog result;
|
||||
|
||||
Matcher matcher = PATCH_LOG_PATTERN.matcher(line);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = new PrivatebinPatchLog();
|
||||
result.setTime(LocalDateTime.parse(matcher.group("datetime"), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withLocale(Locale.ENGLISH)));
|
||||
result.setAction(PrivatebinLogAction.of(matcher.group("action")));
|
||||
result.setId(matcher.group("id"));
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param file
|
||||
* the file
|
||||
* @return the path counters
|
||||
*/
|
||||
public static PathCounters probe(final File file) throws IOException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
|
||||
|
||||
result = new PathCounters();
|
||||
|
||||
//
|
||||
int lineCount = 0;
|
||||
LineIterator iterator = new LineIterator(file);
|
||||
while (iterator.hasNext())
|
||||
{
|
||||
lineCount += 1;
|
||||
String line = iterator.next();
|
||||
|
||||
try
|
||||
{
|
||||
PrivatebinPatchLog log = parseLog(line);
|
||||
// System.out.println("====> " + log);
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
else
|
||||
{
|
||||
probeLog(log);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
System.out.println("PrivatebinPatchLog line count=" + lineCount);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe log.
|
||||
*
|
||||
* @param log
|
||||
* the log
|
||||
*/
|
||||
public static PathCounters probeLog(final PrivatebinPatchLog log)
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
result = new PathCounters();
|
||||
|
||||
// logger.info("==================");
|
||||
if (log != null)
|
||||
{
|
||||
// logger.info("LINE IS MATCHING [{}]", log);
|
||||
|
||||
// Timemarks.
|
||||
String year = log.getYear();
|
||||
String yearMonth = log.getYearMonth();
|
||||
String yearWeek = log.getYearWeek();
|
||||
String date = log.getDate();
|
||||
|
||||
// metrics.pastebins.created
|
||||
// metrics.pastebins.reads
|
||||
// metrics.pastebins.deleted
|
||||
// metrics.pastebins.comment
|
||||
switch (log.getAction())
|
||||
{
|
||||
case COMMENT:
|
||||
result.inc("metrics.pastebins.comment", year, yearMonth, yearWeek, date);
|
||||
break;
|
||||
|
||||
case CREATE:
|
||||
result.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
|
||||
break;
|
||||
|
||||
case DELETE:
|
||||
result.inc("metrics.pastebins.delete", year, yearMonth, yearWeek, date);
|
||||
break;
|
||||
|
||||
case READ:
|
||||
result.inc("metrics.pastebins.reads", year, yearMonth, yearWeek, date);
|
||||
break;
|
||||
|
||||
default:
|
||||
System.out.println("UNKNOWN Privatebin patch log ation!");
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -26,6 +26,7 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounter;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.metrics.UserCounters;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
|
||||
|
|
@ -48,6 +49,19 @@ public class PrivatebinProber
|
|||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param httpAccessLogs
|
||||
* the http access logs
|
||||
* @param httpRegex
|
||||
* the http regex
|
||||
* @return the path counters
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
|
||||
{
|
||||
PathCounters result;
|
||||
|
|
@ -90,7 +104,7 @@ public class PrivatebinProber
|
|||
}
|
||||
|
||||
// metrics.pastebins.created
|
||||
// metrics.pastebins.read
|
||||
// metrics.pastebins.reads
|
||||
// metrics.pastebins.deleted
|
||||
if (StringUtils.startsWith(log.getRequest(), "POST "))
|
||||
{
|
||||
|
|
@ -104,7 +118,7 @@ public class PrivatebinProber
|
|||
}
|
||||
else
|
||||
{
|
||||
result.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
|
||||
result.inc("metrics.pastebins.reads", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -141,7 +155,7 @@ public class PrivatebinProber
|
|||
// metrics.service.users.ipv4
|
||||
// metrics.service.users.ipv6
|
||||
// metrics.pastebins.created
|
||||
// metrics.pastebins.read
|
||||
// metrics.pastebins.reads
|
||||
// metrics.pastebins.deleted
|
||||
result = probe(httpLogs, httpLogRegex);
|
||||
|
||||
|
|
@ -154,6 +168,18 @@ public class PrivatebinProber
|
|||
{
|
||||
long count = FilesUtils.searchByWildcard(datafileDirectory.getAbsolutePath() + "/??/*").size();
|
||||
result.set(count, "metrics.pastebins.count");
|
||||
|
||||
//
|
||||
File patchLogs = new File(datafileDirectory, "metrics.log");
|
||||
if (patchLogs.exists())
|
||||
{
|
||||
PathCounters data = PrivatebinPatchLogProber.probe(patchLogs);
|
||||
|
||||
for (PathCounter counter : data.values())
|
||||
{
|
||||
result.put(counter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// metrics.pastebins.purged
|
||||
|
|
|
|||
55
src/fr/devinsy/statoolinfos/util/FileLines.java
Normal file
55
src/fr/devinsy/statoolinfos/util/FileLines.java
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
/*
|
||||
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* The Class FileLines.
|
||||
*/
|
||||
public class FileLines implements Iterable<String>
|
||||
{
|
||||
private File source;
|
||||
|
||||
/**
|
||||
* Instantiates a new file lines.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public FileLines(final File source)
|
||||
{
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.lang.Iterable#iterator()
|
||||
*/
|
||||
@Override
|
||||
public Iterator<String> iterator()
|
||||
{
|
||||
Iterator<String> result;
|
||||
|
||||
result = new FilesLineIterator(this.source);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -20,6 +20,7 @@ package fr.devinsy.statoolinfos.util;
|
|||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
|
@ -38,6 +39,36 @@ public class Files extends ArrayList<File>
|
|||
super();
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new files.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public Files(final Collection<? extends File> source)
|
||||
{
|
||||
super();
|
||||
if (source != null)
|
||||
{
|
||||
addAll(source);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new files.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public Files(final File source)
|
||||
{
|
||||
super();
|
||||
if (source != null)
|
||||
{
|
||||
add(source);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new files.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ import org.slf4j.Logger;
|
|||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* The Class WildcardFileLineIterator.
|
||||
* The Class FilesLineIterator.
|
||||
*/
|
||||
public class FilesLineIterator implements Iterator<String>
|
||||
{
|
||||
|
|
@ -37,12 +37,23 @@ public class FilesLineIterator implements Iterator<String>
|
|||
private boolean print;
|
||||
|
||||
/**
|
||||
* Instantiates a new wildcard file line iterator.
|
||||
* Instantiates a new files line iterator.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public FilesLineIterator(final File source)
|
||||
{
|
||||
this.fileIterator = new Files(source).iterator();
|
||||
this.lineIterator = null;
|
||||
setPrintOn();
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new files line iterator.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public FilesLineIterator(final Files source)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue