From a1dd2ba48ffb6e32540f485bf39124fc2e32a946 Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Thu, 23 Feb 2023 19:28:52 +0100 Subject: [PATCH] Added pathFilter parameter in httpAccessLog read (#2, Sleto). --- .../statoolinfos/core/Configuration.java | 17 ++++- .../devinsy/statoolinfos/metrics/Prober.java | 69 +++++++++++------- .../etherpad/EtherpadHttpLogAnalyzer.java | 18 ++--- .../metrics/etherpad/EtherpadProber.java | 38 +++++++--- .../metrics/gitea/GiteaProber.java | 29 ++++---- .../metrics/http/HttpAccessLog.java | 3 +- .../metrics/http/HttpAccessLogAnalyzer.java | 70 +++---------------- .../metrics/http/HttpAccessLogIterator.java | 68 ++++++++++++++++-- .../metrics/http/HttpAccessLogs.java | 31 ++++++-- .../metrics/libreqr/LibreQRProber.java | 25 +++---- .../metrics/privatebin/PrivatebinProber.java | 25 +++---- 11 files changed, 231 insertions(+), 162 deletions(-) diff --git a/src/fr/devinsy/statoolinfos/core/Configuration.java b/src/fr/devinsy/statoolinfos/core/Configuration.java index 7a0295c..29dfa8e 100644 --- a/src/fr/devinsy/statoolinfos/core/Configuration.java +++ b/src/fr/devinsy/statoolinfos/core/Configuration.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2022 Christian Pierre MOMON + * Copyright (C) 2020-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -352,6 +352,21 @@ public class Configuration extends PathPropertyList return result; } + /** + * Gets the probe http access log path filter. + * + * @return the probe http access log path filter + */ + public String getProbeHttpAccessLogPathFilter() + { + String result; + + result = get("conf.probe.httpaccesslog.pathfilter"); + + // + return result; + } + /** * Gets the probe http access log pattern. * diff --git a/src/fr/devinsy/statoolinfos/metrics/Prober.java b/src/fr/devinsy/statoolinfos/metrics/Prober.java index 4bdca7d..c3617c4 100644 --- a/src/fr/devinsy/statoolinfos/metrics/Prober.java +++ b/src/fr/devinsy/statoolinfos/metrics/Prober.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2022 Christian Pierre MOMON + * Copyright (C) 2020-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -42,6 +42,7 @@ import fr.devinsy.statoolinfos.metrics.etherpad.EtherpadProber; import fr.devinsy.statoolinfos.metrics.gitea.GiteaProber; import fr.devinsy.statoolinfos.metrics.gsl.GSLProber; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer; import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber; import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber; @@ -116,11 +117,15 @@ public class Prober { logger.info("== Probing HttpAccessLog."); String source = configuration.getProbeHttpAccessLogSource(); - String patternRegex = configuration.getProbeHttpAccessLogPattern(); + String pattern = configuration.getProbeHttpAccessLogPattern(); + String pathFilter = configuration.getProbeHttpAccessLogPathFilter(); logger.info("source=[{}]", source); - logger.info("pattern=[{}]", patternRegex); + logger.info("pattern=[{}]", pattern); + logger.info("path=[{}]", pathFilter); - PathCounters data = HttpAccessLogAnalyzer.probe(source, patternRegex); + HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, pathFilter); + + PathCounters data = HttpAccessLogAnalyzer.probe(httpAccessLogs); counters.putAll(data); } @@ -139,16 +144,20 @@ public class Prober if (types.containsAnyIgnoreCase("Etherpad")) { logger.info("== Probing Etherpad."); - String httpLogs = configuration.getProbeHttpAccessLogSource(); - logger.info("httpLogs=[{}]", httpLogs); String logs = configuration.get("conf.probe.etherpad.logs"); logger.info("logs=[{}]", logs); - String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); - logger.info("httpAccessPattern=[{}]", httpAccessLogRegex); DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.etherpad"); logger.info("database={}", database.toString()); - PathCounters metrics = EtherpadProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, FilesUtils.searchByWildcard(logs), database); + String httpLogs = configuration.getProbeHttpAccessLogSource(); + String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern(); + String httpAccessPathFilter = configuration.getProbeHttpAccessLogPathFilter(); + logger.info("httpLogs=[{}]", httpLogs); + logger.info("httpAccessPattern=[{}]", httpAccessLogPattern); + logger.info("httpAccessPathFilter=[{}]", httpAccessPathFilter); + HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessPathFilter); + + PathCounters metrics = EtherpadProber.probe(httpAccessLogs, FilesUtils.searchByWildcard(logs), database); counters.putAll(metrics); } @@ -168,10 +177,6 @@ public class Prober if (types.containsAnyIgnoreCase("Gitea")) { logger.info("== Probing Gitea."); - String httpLogs = configuration.getProbeHttpAccessLogSource(); - logger.info("httpLogs=[{}]", httpLogs); - String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); - logger.info("httpAccessPattern=[{}]", httpAccessLogRegex); File dataDirectory = configuration.getAsFile("conf.probe.gitea.data"); logger.info("dataPath=[{}]", dataDirectory); String apiURL = configuration.get("conf.probe.gitea.api.url"); @@ -181,7 +186,15 @@ public class Prober DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.gitea"); logger.info("database={}", database.toString()); - PathCounters metrics = GiteaProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, apiURL, apiToken, dataDirectory, database); + String httpLogs = configuration.getProbeHttpAccessLogSource(); + String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern(); + String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter(); + logger.info("httpLogs=[{}]", httpLogs); + logger.info("httpAccessPattern=[{}]", httpAccessLogPattern); + logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter); + HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessLogPathFilter); + + PathCounters metrics = GiteaProber.probe(httpAccessLogs, apiURL, apiToken, dataDirectory, database); counters.putAll(metrics); } @@ -202,16 +215,18 @@ public class Prober { logger.info("== Probing LibreQR."); - String source = configuration.getProbeHttpAccessLogSource(); - logger.info("source=[{}]", source); - - String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); - logger.info("pattern=[{}]", httpAccessLogRegex); + String httpAccessSource = configuration.getProbeHttpAccessLogSource(); + String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern(); + String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter(); + logger.info("httpAccessSource=[{}]", httpAccessSource); + logger.info("httpAccessPattern=[{}]", httpAccessLogPattern); + logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter); + HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessSource), httpAccessLogPattern, httpAccessLogPathFilter); File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles"); logger.info("dataDirectory=[{}]", dataDirectory); - PathCounters data = LibreQRProber.probe(FilesUtils.searchByWildcard(source), httpAccessLogRegex, dataDirectory); + PathCounters data = LibreQRProber.probe(httpAccessLogs, dataDirectory); counters.putAll(data); } @@ -245,14 +260,18 @@ public class Prober if (types.containsAnyIgnoreCase("PrivateBin")) { logger.info("== Probing Privatebin."); - String httpAccessLogs = configuration.getProbeHttpAccessLogSource(); - logger.info("source=[{}]", httpAccessLogs); - String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); - logger.info("pattern=[{}]", httpAccessLogRegex); + String httpAccessLogSource = configuration.getProbeHttpAccessLogSource(); + String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern(); + String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter(); + logger.info("source=[{}]", httpAccessLogSource); + logger.info("pattern=[{}]", httpAccessLogPattern); + logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter); + HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessLogSource), httpAccessLogPattern, httpAccessLogPathFilter); + File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data"); logger.info("dataDirectory=[{}]", dataDirectory); - PathCounters data = PrivatebinProber.probe(FilesUtils.searchByWildcard(httpAccessLogs), httpAccessLogRegex, dataDirectory); + PathCounters data = PrivatebinProber.probe(httpAccessLogs, dataDirectory); counters.putAll(data); } diff --git a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java index 3a96250..8c7d35e 100644 --- a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java +++ b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Christian Pierre MOMON + * Copyright (C) 2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -28,8 +28,7 @@ import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.metrics.PathCounters; import fr.devinsy.statoolinfos.metrics.UserCounters; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator; -import fr.devinsy.statoolinfos.util.Files; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; /** * The Class EtherpadHttpLogAnalyzer. @@ -116,26 +115,23 @@ public class EtherpadHttpLogAnalyzer /** * Probe. * - * @param httpAccessLogFiles - * the http access log files - * @param httpRegex - * the http regex + * @param logs + * the logs * @return the path counters * @throws IOException * Signals that an I/O exception has occurred. * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException + public static PathCounters probe(final HttpAccessLogs logs) throws IOException, StatoolInfosException { PathCounters result; EtherpadHttpLogAnalyzer analyzer = new EtherpadHttpLogAnalyzer(); - HttpAccessLogIterator logs = new HttpAccessLogIterator(httpAccessLogFiles, httpRegex); - while (logs.hasNext()) + for (HttpAccessLog log : logs) { - analyzer.probeLog(logs.next()); + analyzer.probeLog(log); } result = analyzer.getCounters(); diff --git a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java index 9828d31..cac77f1 100644 --- a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Christian Pierre MOMON + * Copyright (C) 2022-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory; import fr.devinsy.statoolinfos.core.DatabaseConfig; import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.metrics.PathCounters; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; import fr.devinsy.statoolinfos.metrics.util.DatabaseProber; import fr.devinsy.statoolinfos.util.Files; import fr.devinsy.statoolinfos.util.sql.SQLDatabase; @@ -49,19 +50,19 @@ public class EtherpadProber /** * Probe. * - * @param httpLogs - * the http logs - * @param httpLogRegex - * the http log regex - * @param dataPath - * the data path + * @param httpAccessLogs + * the http access logs + * @param logs + * the logs + * @param databaseConfig + * the database config * @return the path counters * @throws IOException * Signals that an I/O exception has occurred. * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final Files logs, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException + public static PathCounters probe(final HttpAccessLogs httpAccessLogs, final Files logs, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException { PathCounters result; @@ -70,7 +71,7 @@ public class EtherpadProber // metrics.service.users.ipv6 if (logs.isEmpty()) { - result = EtherpadHttpLogAnalyzer.probe(httpLogs, httpLogRegex); + result = EtherpadHttpLogAnalyzer.probe(httpAccessLogs); } else { @@ -145,6 +146,25 @@ public class EtherpadProber { System.out.println("Etherpad Database undefined."); } + + // https://etherpad.org/doc/v1.8.18/#index_statistics + // Etherpad keeps track of the goings-on inside the edit machinery. + // If you'd like to have a look at this, just point your browser to + // /stats. + // We currently measure: + // + // totalUsers (counter) + // connects (meter) + // disconnects (meter) + // pendingEdits (counter) + // edits (timer) + // failedChangesets (meter) + // httpRequests (timer) + // http500 (meter) + // memoryUsage (gauge) + // + // https://pad.libre-service.eu/stats/ + } catch (SQLException exception) { diff --git a/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java b/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java index 171982a..58a3840 100644 --- a/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Christian Pierre MOMON + * Copyright (C) 2022-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -37,7 +37,6 @@ import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; import fr.devinsy.statoolinfos.metrics.http.HttpStatusCategory; import fr.devinsy.statoolinfos.metrics.util.DatabaseProber; import fr.devinsy.statoolinfos.metrics.util.DatafilesProber; -import fr.devinsy.statoolinfos.util.Files; import fr.devinsy.statoolinfos.util.sql.SQLDatabase; import fr.devinsy.strings.StringList; @@ -58,19 +57,23 @@ public class GiteaProber /** * Probe. * - * @param httpLogs - * the http logs - * @param httpLogRegex - * the http log regex + * @param httpAccessLogs + * the http access logs + * @param apiURL + * the api URL + * @param apiToken + * the api token * @param dataPath * the data path + * @param databaseConfig + * the database config * @return the path counters * @throws IOException * Signals that an I/O exception has occurred. * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final String apiURL, final String apiToken, final File dataPath, final DatabaseConfig databaseConfig) + public static PathCounters probe(final HttpAccessLogs httpAccessLogs, final String apiURL, final String apiToken, final File dataPath, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException { PathCounters result; @@ -78,7 +81,7 @@ public class GiteaProber // metrics.service.users // metrics.service.users.ipv4 // metrics.service.users.ipv6 - result = probeHttpAccessLog(httpLogs, httpLogRegex); + result = probeHttpAccessLog(httpAccessLogs); // metrics.service.database.bytes try @@ -165,17 +168,15 @@ public class GiteaProber /** * Probe http access log. * - * @param httpAccessLogFiles - * the http access log files - * @param httpRegex - * the http regex + * @param logs + * the logs * @return the path counters * @throws IOException * Signals that an I/O exception has occurred. * @throws StatoolInfosException * the statool infos exception */ - private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException + private static PathCounters probeHttpAccessLog(final HttpAccessLogs logs) throws IOException, StatoolInfosException { PathCounters result; @@ -295,7 +296,7 @@ public class GiteaProber Pattern USE_PATTERN = Pattern.compile(regex); // - for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex)) + for (HttpAccessLog log : logs) { // General HTTP access logs. String year = log.getYear(); diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java index d56d13f..ef102ac 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2022 Christian Pierre MOMON + * Copyright (C) 2021-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -33,6 +33,7 @@ import fr.devinsy.strings.StringList; /** * The Class HttpAccessLog. + * */ public class HttpAccessLog { diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java index 7520c12..8dc62ab 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java @@ -18,7 +18,6 @@ */ package fr.devinsy.statoolinfos.metrics.http; -import java.io.File; import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -30,8 +29,6 @@ import org.slf4j.LoggerFactory; import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.metrics.IpCounters; import fr.devinsy.statoolinfos.metrics.PathCounters; -import fr.devinsy.statoolinfos.util.FilesUtils; -import fr.devinsy.statoolinfos.util.LineIterator; /** * The Class HttpAccessLogProber. @@ -40,6 +37,7 @@ public class HttpAccessLogAnalyzer { private static Logger logger = LoggerFactory.getLogger(HttpAccessLogAnalyzer.class); + private int ignoredLineCount; private int errorCount; private PathCounters counters; private IpCounters ips; @@ -64,6 +62,9 @@ public class HttpAccessLogAnalyzer */ public HttpAccessLogAnalyzer() { + this.errorCount = 0; + this.ignoredLineCount = 0; + this.counters = new PathCounters(); this.ips = new IpCounters(); this.ipv4 = new IpCounters(); @@ -122,58 +123,6 @@ public class HttpAccessLogAnalyzer return result; } - /** - * Probe. - * - * @param file - * the file - * @throws IOException - * Signals that an I/O exception has occurred. - * @throws StatoolInfosException - * the statool infos exception - */ - public void probe(final File file, final String patternRegex) throws IOException - { - System.out.println("Probing file [" + file.getAbsolutePath() + "]"); - - // - Pattern pattern; - if (StringUtils.isBlank(patternRegex)) - { - pattern = HttpAccessLogParser.COMBINED_PATTERN; - } - else - { - pattern = Pattern.compile(patternRegex); - } - - // - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) - { - String line = iterator.next(); - - try - { - HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern); - if (log == null) - { - logger.warn("LINE IS NOT MATCHING [{}]", line); - } - else - { - probeLog(log); - } - } - catch (Exception exception) - { - logger.warn("Error parsing line [{}][{}]", line, exception.getMessage()); - exception.printStackTrace(); - this.errorCount += 1; - } - } - } - /** * Probe log. * @@ -404,22 +353,23 @@ public class HttpAccessLogAnalyzer /** * Probe. * - * @param source - * the source + * @param logs + * the logs + * @return the path counters * @throws IOException * Signals that an I/O exception has occurred. * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final String source, final String patternRegex) throws IOException, StatoolInfosException + public static PathCounters probe(final HttpAccessLogs logs) throws IOException, StatoolInfosException { PathCounters result; HttpAccessLogAnalyzer analyzer = new HttpAccessLogAnalyzer(); - for (File file : FilesUtils.searchByWildcard(source)) + for (HttpAccessLog log : logs) { - analyzer.probe(file, patternRegex); + analyzer.probeLog(log); } result = analyzer.getCounters(); diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java index ad71390..bb741e3 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Christian Pierre MOMON + * Copyright (C) 2022-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,7 +38,10 @@ public class HttpAccessLogIterator implements Iterator private FilesLineIterator lineIterator; private Pattern pattern; + private Pattern requestFilter; private HttpAccessLog nextLog; + private int errorCount; + private int ignoredLineCount; /** * Instantiates a new http access log iterator. @@ -49,7 +53,22 @@ public class HttpAccessLogIterator implements Iterator */ public HttpAccessLogIterator(final Files source) throws IOException { - this(source, null); + this(source, null, null); + } + + /** + * Instantiates a new http access log iterator. + * + * @param source + * the source + * @param regex + * the regex + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public HttpAccessLogIterator(final Files source, final String regex) throws IOException + { + this(source, regex, null); } /** @@ -60,12 +79,14 @@ public class HttpAccessLogIterator implements Iterator * @throws IOException * Signals that an I/O exception has occurred. */ - public HttpAccessLogIterator(final Files source, final String regex) throws IOException + public HttpAccessLogIterator(final Files source, final String regex, final String pathFilter) throws IOException { this.lineIterator = new FilesLineIterator(source); this.nextLog = null; + this.errorCount = 0; + this.ignoredLineCount = 0; - if (regex == null) + if (StringUtils.isBlank(regex)) { this.pattern = HttpAccessLogParser.COMBINED_PATTERN; } @@ -73,6 +94,37 @@ public class HttpAccessLogIterator implements Iterator { this.pattern = Pattern.compile(regex); } + + if (StringUtils.isBlank(pathFilter)) + { + this.requestFilter = null; + } + else + { + String filter = "^\\S+ " + pathFilter + " .*$"; + System.out.println("requestFilter=" + filter); + this.requestFilter = Pattern.compile(filter); + } + } + + /** + * Gets the error count. + * + * @return the error count + */ + public int getErrorCount() + { + return this.errorCount; + } + + /** + * Gets the ignored line count. + * + * @return the ignored line count + */ + public int getIgnoredLineCount() + { + return this.ignoredLineCount; } /* (non-Javadoc) @@ -135,12 +187,18 @@ public class HttpAccessLogIterator implements Iterator if (log == null) { logger.warn("LINE IS NOT MATCHING [{}]", line); + this.errorCount += 1; } - else + else if ((this.requestFilter == null) || (this.requestFilter.matcher(log.getRequest()).matches())) { this.nextLog = log; ended = true; } + else + { + this.ignoredLineCount += 1; + // System.out.println("XX " + log.getRequest()); + } } catch (Exception exception) { diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java index b933ad7..38306e1 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Christian Pierre MOMON + * Copyright (C) 2022-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -34,7 +34,8 @@ public class HttpAccessLogs implements Iterable private static Logger logger = LoggerFactory.getLogger(HttpAccessLogs.class); private Files source; - private String regex; + private String pattern; + private String pathFilter; /** * Instantiates a new http access logs. @@ -46,7 +47,7 @@ public class HttpAccessLogs implements Iterable */ public HttpAccessLogs(final Files source) throws IOException { - this(source, null); + this(source, null, null); } /** @@ -57,10 +58,28 @@ public class HttpAccessLogs implements Iterable * @throws IOException * Signals that an I/O exception has occurred. */ - public HttpAccessLogs(final Files source, final String regex) throws IOException + public HttpAccessLogs(final Files source, final String pattern) throws IOException + { + this(source, pattern, null); + } + + /** + * Instantiates a new http access logs. + * + * @param source + * the source + * @param pattern + * the pattern + * @param pathFilter + * the path filter + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public HttpAccessLogs(final Files source, final String pattern, final String pathFilter) throws IOException { this.source = source; - this.regex = regex; + this.pattern = pattern; + this.pathFilter = pathFilter; } /* (non-Javadoc) @@ -73,7 +92,7 @@ public class HttpAccessLogs implements Iterable try { - result = new HttpAccessLogIterator(this.source, this.regex); + result = new HttpAccessLogIterator(this.source, this.pattern, this.pathFilter); } catch (IOException exception) { diff --git a/src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java b/src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java index 8bbb885..3830f66 100644 --- a/src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Christian Pierre MOMON + * Copyright (C) 2022-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -31,7 +31,6 @@ import fr.devinsy.statoolinfos.metrics.UserCounters; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; import fr.devinsy.statoolinfos.metrics.util.DatafilesProber; -import fr.devinsy.statoolinfos.util.Files; /** * The Class LibreQRProber. @@ -50,10 +49,8 @@ public class LibreQRProber /** * Probe. * - * @param httpLogs - * the http logs - * @param httpLogRegex - * the http log regex + * @param httpAccessLogs + * the http access logs * @param dataPath * the data path * @return the path counters @@ -62,7 +59,7 @@ public class LibreQRProber * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException + public static PathCounters probe(final HttpAccessLogs httpAccessLogs, final File dataPath) throws IOException, StatoolInfosException { PathCounters result; @@ -71,7 +68,7 @@ public class LibreQRProber // metrics.service.users.ipv6 // metrics.barcodes.created // metrics.libreqr.barcodes.downloads - result = probeHttpAccessLog(httpLogs, httpLogRegex); + result = probeHttpAccessLog(httpAccessLogs); // metrics.service.files.bytes, metrics.libreqr.cache.bytes // metrics.service.files.count, metrics.libreqr.cache.count @@ -82,19 +79,17 @@ public class LibreQRProber } /** - * Probe. + * Probe http access log. * - * @param httpAccessLogFiles - * the http access log files - * @param httpRegex - * the http regex + * @param logs + * the logs * @return the path counters * @throws IOException * Signals that an I/O exception has occurred. * @throws StatoolInfosException * the statool infos exception */ - private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException + private static PathCounters probeHttpAccessLog(final HttpAccessLogs logs) throws IOException, StatoolInfosException { PathCounters result; @@ -109,7 +104,7 @@ public class LibreQRProber Pattern CREATE_PATTERN = Pattern.compile("POST / .*"); // - for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex)) + for (HttpAccessLog log : logs) { // General HTTP access logs. String year = log.getYear(); diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java index ebba90b..b48d930 100644 --- a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Christian Pierre MOMON + * Copyright (C) 2021-2023 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -32,7 +32,6 @@ import fr.devinsy.statoolinfos.metrics.UserCounters; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; import fr.devinsy.statoolinfos.metrics.util.DatafilesProber; -import fr.devinsy.statoolinfos.util.Files; import fr.devinsy.statoolinfos.util.FilesUtils; /** @@ -52,17 +51,15 @@ public class PrivatebinProber /** * Probe. * - * @param httpAccessLogs - * the http access logs - * @param httpRegex - * the http regex + * @param logs + * the logs * @return the path counters * @throws IOException * Signals that an I/O exception has occurred. * @throws StatoolInfosException * the statool infos exception */ - private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException + private static PathCounters probe(final HttpAccessLogs logs) throws IOException, StatoolInfosException { PathCounters result; @@ -74,7 +71,7 @@ public class PrivatebinProber UserCounters ipv6Users = new UserCounters(); // - for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogs, httpRegex)) + for (HttpAccessLog log : logs) { // logger.info("LINE IS MATCHING [{}]", log); // logger.info(log.getHttpUserAgent().toString()); @@ -135,19 +132,17 @@ public class PrivatebinProber /** * Probe. * - * @param httpLogs - * the http logs - * @param httpLogRegex - * the http log regex + * @param httpAccessLogs + * the http access logs * @param datafileDirectory - * the data path + * the datafile directory * @return the path counters * @throws IOException * Signals that an I/O exception has occurred. * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File datafileDirectory) throws IOException, StatoolInfosException + public static PathCounters probe(final HttpAccessLogs httpAccessLogs, final File datafileDirectory) throws IOException, StatoolInfosException { PathCounters result; @@ -157,7 +152,7 @@ public class PrivatebinProber // metrics.pastebins.created // metrics.pastebins.reads // metrics.pastebins.deleted - result = probe(httpLogs, httpLogRegex); + result = probe(httpAccessLogs); // metrics.service.files.bytes // metrics.service.files.count