From 50ec907ac8655f8bdbaa8b5ed9fefde0a9591b35 Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Fri, 31 Dec 2021 14:05:49 +0100 Subject: [PATCH] Added PrivateBin probing. --- .../privatebin/PrivatebinDataAnalyzer.java | 90 ++++++++ .../privatebin/PrivatebinHttpLogAnalyzer.java | 215 ++++++++++++++++++ .../metrics/privatebin/PrivatebinProber.java | 76 +++++++ 3 files changed, 381 insertions(+) create mode 100644 src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java create mode 100644 src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java create mode 100644 src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java new file mode 100644 index 0000000..40d9446 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.privatebin; + +import java.io.File; +import java.io.IOException; +import java.time.LocalDate; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.metrics.PathCounters; +import fr.devinsy.statoolinfos.metrics.TimeMark; +import fr.devinsy.statoolinfos.util.FilesUtils; + +/** + * The Class PrivatebinDataAnalyzer. + */ +public class PrivatebinDataAnalyzer +{ + private static Logger logger = LoggerFactory.getLogger(PrivatebinDataAnalyzer.class); + + /** + * Instantiates a new http access log prober. + */ + private PrivatebinDataAnalyzer() + { + } + + /** + * Probe. + * + * @param httpAccessLogs + * the source + * @throws IOException + * Signals that an I/O exception has occurred. + * @throws StatoolInfosException + * the statool infos exception + */ + public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException + { + PathCounters result; + + System.out.println("Probing directory [" + dataDirectory + "]"); + + result = new PathCounters(); + + if ((dataDirectory.exists()) && (dataDirectory.isDirectory())) + { + LocalDate now = LocalDate.now(); + String year = TimeMark.yearOf(now).toString(); + String yearMonth = TimeMark.yearMonthOf(now).toString(); + String yearWeek = TimeMark.yearWeekOf(now).toString(); + String date = TimeMark.dayOf(now).toString(); + + // metrics.service.files.bytes + long size = FileUtils.sizeOfDirectory(dataDirectory); + result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date); + + // metrics.pastebins.count + long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size(); + result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date); + } + else + { + System.out.println("WARNING: Privatebin data path is not valid."); + } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java new file mode 100644 index 0000000..a149637 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java @@ -0,0 +1,215 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.privatebin; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.metrics.PathCounters; +import fr.devinsy.statoolinfos.metrics.UserCounters; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer; +import fr.devinsy.statoolinfos.util.FilesUtils; +import fr.devinsy.statoolinfos.util.LineIterator; + +/** + * The Class HttpAccessLogProber. + */ +public class PrivatebinHttpLogAnalyzer +{ + private static Logger logger = LoggerFactory.getLogger(PrivatebinHttpLogAnalyzer.class); + + private PathCounters counters; + private UserCounters users; + private UserCounters ipv4Users; + private UserCounters ipv6Users; + + /** + * Instantiates a new http access log prober. + */ + public PrivatebinHttpLogAnalyzer() + { + this.counters = new PathCounters(); + this.users = new UserCounters(); + this.ipv4Users = new UserCounters(); + this.ipv6Users = new UserCounters(); + } + + /** + * Gets the counters. + * + * @return the counters + */ + public PathCounters getCounters() + { + PathCounters result; + + result = new PathCounters(); + result.putAll(this.counters); + + result.putAll(this.users.getCounters("metrics.service.users")); + result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4")); + result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6")); + + // + return result; + } + + /** + * Probe. + * + * @param file + * the file + * @throws IOException + * Signals that an I/O exception has occurred. + * @throws StatoolInfosException + * the statool infos exception + */ + public void probe(final File file, final String patternRegex) throws IOException + { + System.out.println("Probing file [" + file.getAbsolutePath() + "]"); + + // + Pattern pattern; + if (patternRegex == null) + { + pattern = HttpAccessLogAnalyzer.COMBINED_PATTERN; + } + else + { + pattern = Pattern.compile(patternRegex); + } + + // + LineIterator iterator = new LineIterator(file); + while (iterator.hasNext()) + { + String line = iterator.next(); + + try + { + HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, pattern); + if (log == null) + { + logger.warn("LINE IS NOT MATCHING [{}]", line); + } + else + { + probeLog(log); + } + } + catch (Exception exception) + { + logger.warn("Error parsing line [{}][{}]", line, exception.getMessage()); + exception.printStackTrace(); + } + } + } + + /** + * Probe log. + * + * @param log + * the log + */ + public void probeLog(final HttpAccessLog log) + { + if (log != null) + { + // logger.info("LINE IS MATCHING [{}]", log); + // logger.info(log.getHttpUserAgent().toString()); + + // General HTTP access logs. + String year = log.getYear(); + String yearMonth = log.getYearMonth(); + String yearWeek = log.getYearWeek(); + String date = log.getDate(); + + // metrics.service.users + // metrics.service.users.ipv4 + // metrics.service.users.ipv6 + if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid=")) + { + String key = String.format("%s---%s", log.getIp(), log.getUserAgent()); + this.users.put(key, year, yearMonth, yearWeek, date); + + if (log.isIPv4()) + { + this.ipv4Users.put(key, year, yearMonth, yearWeek, date); + } + else + { + this.ipv6Users.put(key, year, yearMonth, yearWeek, date); + } + } + + // metrics.pastebins.created + // metrics.pastebins.read + // metrics.pastebins.deleted + if (StringUtils.startsWith(log.getRequest(), "POST ")) + { + this.counters.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date); + } + else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid=")) + { + if (StringUtils.contains(log.getRequest(), "&deletetoken=")) + { + this.counters.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date); + } + else + { + this.counters.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date); + } + } + } + } + + /** + * Probe. + * + * @param httpAccessLogs + * the source + * @throws IOException + * Signals that an I/O exception has occurred. + * @throws StatoolInfosException + * the statool infos exception + */ + public static PathCounters probe(final String httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException + { + PathCounters result; + + PrivatebinHttpLogAnalyzer analyzer = new PrivatebinHttpLogAnalyzer(); + + for (File file : FilesUtils.searchByWildcard(httpAccessLogs)) + { + analyzer.probe(file, httpRegex); + } + + result = analyzer.getCounters(); + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java new file mode 100644 index 0000000..52482f4 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.privatebin; + +import java.io.File; +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.metrics.PathCounters; + +/** + * The Class PrivatebinProber. + */ +public class PrivatebinProber +{ + private static Logger logger = LoggerFactory.getLogger(PrivatebinProber.class); + + /** + * Instantiates a new privatebin prober. + */ + public PrivatebinProber() + { + } + + /** + * Probe. + * + * @param logs + * the logs + * @param databaseConfig + * the database config + * @throws StatoolInfosException + * @throws IOException + */ + public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException + { + PathCounters result; + + // metrics.service.users + // metrics.service.users.ipv4 + // metrics.service.users.ipv6 + // metrics.pastebins.created + // metrics.pastebins.read + // metrics.pastebins.deleted + result = PrivatebinHttpLogAnalyzer.probe(httpLogs, httpLogRegex); + + // metrics.service.files.bytes + // metrics.pastebins.count + result.putAll(PrivatebinDataAnalyzer.probe(dataPath)); + + // metrics.pastebins.purged + // purged = count(n-1) - count(n) + created - delete + + // + return result; + } +}