From 950d06fae7826a12ac41a4753b570d1e6ae305fa Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Fri, 11 Mar 2022 14:56:16 +0100 Subject: [PATCH] Added Privatebin patch log probing. --- README.md | 4 +- .../privatebin/PrivatebinLogAction.java | 68 +++++++ .../privatebin/PrivatebinPatchLog.java | 172 +++++++++++++++++ .../privatebin/PrivatebinPatchLogProber.java | 182 ++++++++++++++++++ .../metrics/privatebin/PrivatebinProber.java | 32 ++- .../devinsy/statoolinfos/util/FileLines.java | 55 ++++++ src/fr/devinsy/statoolinfos/util/Files.java | 31 +++ .../statoolinfos/util/FilesLineIterator.java | 19 +- 8 files changed, 554 insertions(+), 9 deletions(-) create mode 100644 src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinLogAction.java create mode 100644 src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinPatchLog.java create mode 100644 src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinPatchLogProber.java create mode 100644 src/fr/devinsy/statoolinfos/util/FileLines.java diff --git a/README.md b/README.md index f39b2f9..20c2d44 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ conf.probe.gitea.database.password= conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties ``` -### LibreQR metrics (coming soon) +### LibreQR metrics Configuration template: @@ -211,7 +211,7 @@ conf.probe.httpaccesslog.pattern= conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties ``` -### PrivateBin metrics (experimental) +### PrivateBin metrics (partial) Warning: works fine if database, image and comment options are disabled. diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinLogAction.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinLogAction.java new file mode 100644 index 0000000..d326f8e --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinLogAction.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.privatebin; + +import org.apache.commons.lang3.StringUtils; + +/** + * The Enum PrivatebinLogAction. + */ +public enum PrivatebinLogAction +{ + COMMENT, + CREATE, + DELETE, + READ; + + /** + * Of. + * + * @param value + * the value + * @return the privatebin log action + */ + public static PrivatebinLogAction of(final String value) + { + PrivatebinLogAction result; + + if (StringUtils.equals(value, "COMMENT")) + { + result = COMMENT; + } + else if (StringUtils.equals(value, "CREATE")) + { + result = CREATE; + } + else if (StringUtils.equals(value, "DELETE")) + { + result = DELETE; + } + else if (StringUtils.equals(value, "READ")) + { + result = READ; + } + else + { + result = null; + } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinPatchLog.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinPatchLog.java new file mode 100644 index 0000000..534d857 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinPatchLog.java @@ -0,0 +1,172 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.privatebin; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.metrics.TimeMarkUtils; + +/** + * The Class PrivatebinPatchLog. + */ +public class PrivatebinPatchLog +{ + private static Logger logger = LoggerFactory.getLogger(PrivatebinPatchLog.class); + + private LocalDateTime time; + private PrivatebinLogAction action; + private String id; + + /** + * Instantiates a new privatebin patch log. + */ + public PrivatebinPatchLog() + { + this.time = null; + this.action = null; + this.id = null; + } + + public PrivatebinLogAction getAction() + { + return this.action; + } + + /** + * Gets the date. + * + * @return the date + */ + public String getDate() + { + String result; + + result = this.time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE)); + + // + return result; + } + + public String getId() + { + return this.id; + } + + public LocalDateTime getTime() + { + return this.time; + } + + /** + * Gets the year. + * + * @return the year + */ + public String getYear() + { + String result; + + if (this.time == null) + { + result = null; + } + else + { + result = TimeMarkUtils.yearOf(this.time); + } + + // + return result; + } + + /** + * Gets the year month. + * + * @return the year month + */ + public String getYearMonth() + { + String result; + + if (this.time == null) + { + result = null; + } + else + { + result = TimeMarkUtils.yearMonthOf(this.time); + } + // + return result; + } + + /** + * Gets the year week. + * + * @return the year week + */ + public String getYearWeek() + { + String result; + if (this.time == null) + { + result = null; + } + else + { + result = TimeMarkUtils.yearWeekOf(this.time); + } + // + return result; + } + + public void setAction(final PrivatebinLogAction action) + { + this.action = action; + } + + public void setId(final String id) + { + this.id = id; + } + + public void setTime(final LocalDateTime time) + { + this.time = time; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() + { + String result; + + result = String.format("[time=%s, action=%s, id=%s]", this.time, this.action, this.id); + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinPatchLogProber.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinPatchLogProber.java new file mode 100644 index 0000000..ad120ca --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinPatchLogProber.java @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.privatebin; + +import java.io.File; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.metrics.PathCounters; +import fr.devinsy.statoolinfos.util.LineIterator; + +/** + * The Class PrivatebinPatchLogAnalyzer. + */ +public class PrivatebinPatchLogProber +{ + private static Logger logger = LoggerFactory.getLogger(PrivatebinPatchLogProber.class); + + // 2021-11-17 01:51:59 DELETE 7922dd508e0cf9b2 + public static final Pattern PATCH_LOG_PATTERN = Pattern.compile( + "^(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+(?\\S+)\\s+(?\\S+)$"); + + /** + * Instantiates a new privatebin patch log analyzer. + */ + private PrivatebinPatchLogProber() + { + } + + /** + * Parses the log. + * + * @param line + * the line + * @return the http log + */ + public static PrivatebinPatchLog parseLog(final String line) + { + PrivatebinPatchLog result; + + Matcher matcher = PATCH_LOG_PATTERN.matcher(line); + if (matcher.matches()) + { + result = new PrivatebinPatchLog(); + result.setTime(LocalDateTime.parse(matcher.group("datetime"), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withLocale(Locale.ENGLISH))); + result.setAction(PrivatebinLogAction.of(matcher.group("action"))); + result.setId(matcher.group("id")); + } + else + { + result = null; + } + + // + return result; + } + + /** + * Probe. + * + * @param file + * the file + * @return the path counters + */ + public static PathCounters probe(final File file) throws IOException + { + PathCounters result; + + System.out.println("Probing file [" + file.getAbsolutePath() + "]"); + + result = new PathCounters(); + + // + int lineCount = 0; + LineIterator iterator = new LineIterator(file); + while (iterator.hasNext()) + { + lineCount += 1; + String line = iterator.next(); + + try + { + PrivatebinPatchLog log = parseLog(line); + // System.out.println("====> " + log); + if (log == null) + { + logger.warn("LINE IS NOT MATCHING [{}]", line); + } + else + { + probeLog(log); + } + } + catch (Exception exception) + { + logger.warn("Error parsing line [{}][{}]", line, exception.getMessage()); + exception.printStackTrace(); + } + } + System.out.println("PrivatebinPatchLog line count=" + lineCount); + + // + return result; + } + + /** + * Probe log. + * + * @param log + * the log + */ + public static PathCounters probeLog(final PrivatebinPatchLog log) + { + PathCounters result; + + result = new PathCounters(); + + // logger.info("=================="); + if (log != null) + { + // logger.info("LINE IS MATCHING [{}]", log); + + // Timemarks. + String year = log.getYear(); + String yearMonth = log.getYearMonth(); + String yearWeek = log.getYearWeek(); + String date = log.getDate(); + + // metrics.pastebins.created + // metrics.pastebins.reads + // metrics.pastebins.deleted + // metrics.pastebins.comment + switch (log.getAction()) + { + case COMMENT: + result.inc("metrics.pastebins.comment", year, yearMonth, yearWeek, date); + break; + + case CREATE: + result.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date); + break; + + case DELETE: + result.inc("metrics.pastebins.delete", year, yearMonth, yearWeek, date); + break; + + case READ: + result.inc("metrics.pastebins.reads", year, yearMonth, yearWeek, date); + break; + + default: + System.out.println("UNKNOWN Privatebin patch log ation!"); + } + } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java index 8629e74..ebba90b 100644 --- a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java @@ -26,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.metrics.PathCounter; import fr.devinsy.statoolinfos.metrics.PathCounters; import fr.devinsy.statoolinfos.metrics.UserCounters; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; @@ -48,6 +49,19 @@ public class PrivatebinProber { } + /** + * Probe. + * + * @param httpAccessLogs + * the http access logs + * @param httpRegex + * the http regex + * @return the path counters + * @throws IOException + * Signals that an I/O exception has occurred. + * @throws StatoolInfosException + * the statool infos exception + */ private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException { PathCounters result; @@ -90,7 +104,7 @@ public class PrivatebinProber } // metrics.pastebins.created - // metrics.pastebins.read + // metrics.pastebins.reads // metrics.pastebins.deleted if (StringUtils.startsWith(log.getRequest(), "POST ")) { @@ -104,7 +118,7 @@ public class PrivatebinProber } else { - result.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date); + result.inc("metrics.pastebins.reads", year, yearMonth, yearWeek, date); } } } @@ -141,7 +155,7 @@ public class PrivatebinProber // metrics.service.users.ipv4 // metrics.service.users.ipv6 // metrics.pastebins.created - // metrics.pastebins.read + // metrics.pastebins.reads // metrics.pastebins.deleted result = probe(httpLogs, httpLogRegex); @@ -154,6 +168,18 @@ public class PrivatebinProber { long count = FilesUtils.searchByWildcard(datafileDirectory.getAbsolutePath() + "/??/*").size(); result.set(count, "metrics.pastebins.count"); + + // + File patchLogs = new File(datafileDirectory, "metrics.log"); + if (patchLogs.exists()) + { + PathCounters data = PrivatebinPatchLogProber.probe(patchLogs); + + for (PathCounter counter : data.values()) + { + result.put(counter); + } + } } // metrics.pastebins.purged diff --git a/src/fr/devinsy/statoolinfos/util/FileLines.java b/src/fr/devinsy/statoolinfos/util/FileLines.java new file mode 100644 index 0000000..7c689da --- /dev/null +++ b/src/fr/devinsy/statoolinfos/util/FileLines.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.util; + +import java.io.File; +import java.util.Iterator; + +/** + * The Class FileLines. + */ +public class FileLines implements Iterable +{ + private File source; + + /** + * Instantiates a new file lines. + * + * @param source + * the source + */ + public FileLines(final File source) + { + this.source = source; + } + + /* (non-Javadoc) + * @see java.lang.Iterable#iterator() + */ + @Override + public Iterator iterator() + { + Iterator result; + + result = new FilesLineIterator(this.source); + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/util/Files.java b/src/fr/devinsy/statoolinfos/util/Files.java index 12315c5..13b30c1 100644 --- a/src/fr/devinsy/statoolinfos/util/Files.java +++ b/src/fr/devinsy/statoolinfos/util/Files.java @@ -20,6 +20,7 @@ package fr.devinsy.statoolinfos.util; import java.io.File; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.regex.Pattern; @@ -38,6 +39,36 @@ public class Files extends ArrayList super(); } + /** + * Instantiates a new files. + * + * @param source + * the source + */ + public Files(final Collection source) + { + super(); + if (source != null) + { + addAll(source); + } + } + + /** + * Instantiates a new files. + * + * @param source + * the source + */ + public Files(final File source) + { + super(); + if (source != null) + { + add(source); + } + } + /** * Instantiates a new files. * diff --git a/src/fr/devinsy/statoolinfos/util/FilesLineIterator.java b/src/fr/devinsy/statoolinfos/util/FilesLineIterator.java index d9cf57e..ce8fdfc 100644 --- a/src/fr/devinsy/statoolinfos/util/FilesLineIterator.java +++ b/src/fr/devinsy/statoolinfos/util/FilesLineIterator.java @@ -26,7 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * The Class WildcardFileLineIterator. + * The Class FilesLineIterator. */ public class FilesLineIterator implements Iterator { @@ -37,12 +37,23 @@ public class FilesLineIterator implements Iterator private boolean print; /** - * Instantiates a new wildcard file line iterator. + * Instantiates a new files line iterator. + * + * @param source + * the source + */ + public FilesLineIterator(final File source) + { + this.fileIterator = new Files(source).iterator(); + this.lineIterator = null; + setPrintOn(); + } + + /** + * Instantiates a new files line iterator. * * @param source * the source - * @throws IOException - * Signals that an I/O exception has occurred. */ public FilesLineIterator(final Files source) {