From d453979eef6a805cdd40e761ecfca7dc2743d7de Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Sat, 20 Jul 2024 02:58:27 +0200 Subject: [PATCH] Refactored http error log metrics. Added list errfile and list errlog commands. --- .../statoolinfos/cli/StatoolInfosCLI.java | 40 ++- .../statoolinfos/core/Configuration.java | 50 +++- .../statoolinfos/core/StatoolInfos.java | 146 +++++++++- .../statoolinfos/metrics/PathCounters.java | 27 ++ .../devinsy/statoolinfos/metrics/Prober.java | 26 +- .../metrics/httperrorlog/HttpErrorLog.java | 38 ++- .../httperrorlog/HttpErrorLogAnalyzer.java | 175 ++--------- .../httperrorlog/HttpErrorLogIterator.java | 275 ++++++++++++++++++ .../httperrorlog/HttpErrorLogParser.java | 120 ++++++++ .../metrics/httperrorlog/HttpErrorLogs.java | 153 ++++++++++ .../metrics/httperror/HttpErrorLogsTest.java | 204 +++++++++++++ 11 files changed, 1091 insertions(+), 163 deletions(-) create mode 100644 src/fr/devinsy/statoolinfos/metrics/httperrorlog/HttpErrorLogIterator.java create mode 100644 src/fr/devinsy/statoolinfos/metrics/httperrorlog/HttpErrorLogParser.java create mode 100644 src/fr/devinsy/statoolinfos/metrics/httperrorlog/HttpErrorLogs.java create mode 100644 test/fr/devinsy/statoolinfos/metrics/httperror/HttpErrorLogsTest.java diff --git a/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java b/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java index ef9a0f2..cf461a2 100644 --- a/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java +++ b/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2023 Christian Pierre MOMON + * Copyright (C) 2020-2024 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -124,6 +124,8 @@ public final class StatoolInfosCLI message.appendln(" statoolinfos stat ip [-bot|-nobot] generate stats about ip from http log file"); message.appendln(" statoolinfos stat ua [-bot|-nobot] generate stats about user agent from http log file"); message.appendln(" statoolinfos stat visitor [-bot|-nobot] generate stats about visitor (ip+ua) from http log file"); + message.appendln(" statoolinfos list errfile display http error log files"); + message.appendln(" statoolinfos list errlog display http error log lines"); System.out.print(message.toString()); } @@ -351,7 +353,7 @@ public final class StatoolInfosCLI { File configurationFile = new File(StringUtils.trim(args[2])); - StatoolInfos.listFiles(configurationFile); + StatoolInfos.listAccessFiles(configurationFile); } else if (CLIUtils.isMatchingEllipsis(args, "list", "(file|files)", ".+")) { @@ -361,7 +363,23 @@ public final class StatoolInfosCLI source.add(new File(args[index])); } - StatoolInfos.listFiles(source); + StatoolInfos.listAccessFiles(source); + } + else if (CLIUtils.isMatching(args, "list", "(errfile|errfiles)", ".+\\.conf")) + { + File configurationFile = new File(StringUtils.trim(args[2])); + + StatoolInfos.listErrorFiles(configurationFile); + } + else if (CLIUtils.isMatchingEllipsis(args, "list", "(errfile|errfiles)", ".+")) + { + Files source = new Files(); + for (int index = 2; index < args.length; index++) + { + source.add(new File(args[index])); + } + + StatoolInfos.listErrorFiles(source); } else if (CLIUtils.isMatching(args, "list", "(log|logs)", "(-all|-bot|-nobot)", ".+\\.conf*")) { @@ -397,6 +415,22 @@ public final class StatoolInfosCLI StatoolInfos.listLogs(source, BotFilter.ALL); } + else if (CLIUtils.isMatching(args, "list", "(errlog|errlogs)", ".+\\.conf")) + { + File configurationFile = new File(StringUtils.trim(args[2])); + + StatoolInfos.listErrorLogs(configurationFile); + } + else if (CLIUtils.isMatchingEllipsis(args, "list", "(errlog|errlogs)", ".+")) + { + Files source = new Files(); + for (int index = 2; index < args.length; index++) + { + source.add(new File(args[index])); + } + + StatoolInfos.listErrorLogs(source); + } else if (CLIUtils.isMatching(args, "list", "(useragent|ua)", "(-all|-bot|-nobot)", ".+\\.conf*")) { BotFilter filter = parseLogFilterOption(args[2]); diff --git a/src/fr/devinsy/statoolinfos/core/Configuration.java b/src/fr/devinsy/statoolinfos/core/Configuration.java index a73d26e..963e2ba 100644 --- a/src/fr/devinsy/statoolinfos/core/Configuration.java +++ b/src/fr/devinsy/statoolinfos/core/Configuration.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2023 Christian Pierre MOMON + * Copyright (C) 2020-2024 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -25,6 +25,7 @@ import java.net.URL; import org.apache.commons.lang3.StringUtils; import fr.devinsy.statoolinfos.properties.PathProperties; +import fr.devinsy.statoolinfos.properties.PathProperty; import fr.devinsy.statoolinfos.properties.PathPropertyList; import fr.devinsy.strings.StringList; @@ -54,6 +55,23 @@ public class Configuration extends PathPropertyList super(properties); } + /** + * Adds the. + * + * @param path + * the path + * @param value + * the value + */ + public void add(final String path, final String value) + { + if (!StringUtils.isBlank(path)) + { + PathProperty property = new PathProperty(path, value); + this.add(property); + } + } + /** * Gets the builds the directory. * @@ -443,6 +461,36 @@ public class Configuration extends PathPropertyList return result; } + /** + * Gets the probe http error log date time pattern. + * + * @return the probe http error log date time pattern + */ + public String getProbeHttpErrorLogDateTimePattern() + { + String result; + + result = get("conf.probe.httperrorlog.datetimepattern"); + + // + return result; + } + + /** + * Gets the probe http error log pattern. + * + * @return the probe http error log pattern + */ + public String getProbeHttpErrorLogPattern() + { + String result; + + result = get("conf.probe.httperroorlog.pattern"); + + // + return result; + } + /** * Gets the probe http error log source. * diff --git a/src/fr/devinsy/statoolinfos/core/StatoolInfos.java b/src/fr/devinsy/statoolinfos/core/StatoolInfos.java index 67ce9ad..cf0359f 100644 --- a/src/fr/devinsy/statoolinfos/core/StatoolInfos.java +++ b/src/fr/devinsy/statoolinfos/core/StatoolInfos.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2023 Christian Pierre MOMON + * Copyright (C) 2020-2024 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -33,8 +33,10 @@ import fr.devinsy.statoolinfos.build.Builder; import fr.devinsy.statoolinfos.crawl.Crawler; import fr.devinsy.statoolinfos.htmlize.Htmlizer; import fr.devinsy.statoolinfos.metrics.Prober; -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; +import fr.devinsy.statoolinfos.metrics.httpaccess.HttpAccessLog; +import fr.devinsy.statoolinfos.metrics.httpaccess.HttpAccessLogs; +import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLog; +import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogs; import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathPropertyUtils; import fr.devinsy.statoolinfos.stats.ip.IpStat; @@ -169,7 +171,7 @@ public class StatoolInfos * @param configurationFile * the configuration file */ - public static void listFiles(final File configurationFile) + public static void listAccessFiles(final File configurationFile) { try { @@ -209,7 +211,7 @@ public class StatoolInfos * @param files * the files */ - public static void listFiles(final Files files) + public static void listAccessFiles(final Files files) { if (files != null) { @@ -223,6 +225,140 @@ public class StatoolInfos } } + /** + * List error files. + * + * @param configurationFile + * the configuration file + */ + public static void listErrorFiles(final File configurationFile) + { + try + { + if ((configurationFile == null) || (!configurationFile.exists())) + { + System.out.println("No configuration file found."); + } + else + { + System.out.println("Listing HttpErrorLog files from [" + configurationFile.toString() + "]"); + Configuration configuration = Factory.loadConfiguration(configurationFile); + + logger.info("== List HttpErrorLog files."); + String source = configuration.getProbeHttpErrorLogSource(); + logger.info("source=[{}]", source); + + Files files = FilesUtils.searchByWildcard(source); + for (File file : files) + { + System.out.println(file.getAbsolutePath()); + } + } + } + catch (StatoolInfosException exception) + { + exception.printStackTrace(); + } + catch (IOException exception) + { + exception.printStackTrace(); + } + } + + /** + * List error files. + * + * @param files + * the files + */ + public static void listErrorFiles(final Files files) + { + if (files != null) + { + System.out.println("Listing HttpErrorLog files from [" + files.toString() + "]"); + + logger.info("== List HttpErrorLog files"); + for (File file : files) + { + System.out.println(file.getAbsolutePath()); + } + } + } + + /** + * List error logs. + * + * @param configurationFile + * the configuration file + */ + public static void listErrorLogs(final File configurationFile) + { + try + { + if ((configurationFile == null) || (!configurationFile.exists())) + { + System.out.println("No configuration file found."); + } + else + { + System.out.println("Testing HttpErrorLog lines from [" + configurationFile.toString() + "]"); + Configuration configuration = Factory.loadConfiguration(configurationFile); + + logger.info("== Testing HttpErrorLog lines."); + String source = configuration.getProbeHttpErrorLogSource(); + String dateTimePattern = configuration.getProbeHttpErrorLogDateTimePattern(); + String pattern = configuration.getProbeHttpErrorLogPattern(); + logger.info("source=[{}]", source); + logger.info("pattern=[{}]", pattern); + logger.info("dateTimePattern=[{}]", dateTimePattern); + + HttpErrorLogs logs = new HttpErrorLogs(FilesUtils.searchByWildcard(source), pattern, dateTimePattern); + + for (HttpErrorLog log : logs) + { + System.out.println(log.toStringLog()); + } + } + } + catch (Exception exception) + { + logger.error("Error: {}", exception.getMessage()); + exception.printStackTrace(); + } + } + + /** + * List error logs. + * + * @param source + * the source + */ + public static void listErrorLogs(final Files source) + { + try + { + HttpErrorLogs logs = new HttpErrorLogs(source); + for (HttpErrorLog log : logs) + { + System.out.println(log.toStringLog()); + } + } + catch (IllegalArgumentException exception) + { + System.out.println("Bad format line detected. Aborting…"); + exception.printStackTrace(); + } + catch (DateTimeParseException exception) + { + System.out.println("Bad datetime format detected. Aborting…"); + } + catch (IOException exception) + { + System.out.println("File error detected. Aborting…"); + exception.printStackTrace(); + } + } + /** * List ips. * diff --git a/src/fr/devinsy/statoolinfos/metrics/PathCounters.java b/src/fr/devinsy/statoolinfos/metrics/PathCounters.java index 4948c9d..7d49c21 100644 --- a/src/fr/devinsy/statoolinfos/metrics/PathCounters.java +++ b/src/fr/devinsy/statoolinfos/metrics/PathCounters.java @@ -111,6 +111,33 @@ public class PathCounters extends HashMap return result; } + /** + * Gets the count. + * + * @param path + * the path + * @param timeMark + * the time mark + * @return the count + */ + public long getCount(final String path, final String timeMark) + { + long result; + + PathCounter counter = get(path, timeMark); + if (counter == null) + { + result = 0; + } + else + { + result = counter.getCounter(); + } + + // + return result; + } + /** * Gets the days values line. * diff --git a/src/fr/devinsy/statoolinfos/metrics/Prober.java b/src/fr/devinsy/statoolinfos/metrics/Prober.java index 844b26d..9c0fd27 100644 --- a/src/fr/devinsy/statoolinfos/metrics/Prober.java +++ b/src/fr/devinsy/statoolinfos/metrics/Prober.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2023 Christian Pierre MOMON + * Copyright (C) 2020-2024 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -43,9 +43,10 @@ import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.metrics.etherpad.EtherpadProber; import fr.devinsy.statoolinfos.metrics.gitea.GiteaProber; import fr.devinsy.statoolinfos.metrics.gsl.GSLProber; -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer; -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; +import fr.devinsy.statoolinfos.metrics.httpaccess.HttpAccessLogAnalyzer; +import fr.devinsy.statoolinfos.metrics.httpaccess.HttpAccessLogs; import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogAnalyzer; +import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogs; import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber; import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber; import fr.devinsy.statoolinfos.metrics.mumble.MumbleProber; @@ -371,15 +372,32 @@ public class Prober return result; } + /** + * Probe http error log. + * + * @param configuration + * the configuration + * @return the path counters + * @throws IOException + * Signals that an I/O exception has occurred. + * @throws StatoolInfosException + * the statool infos exception + */ public static PathCounters probeHttpErrorLog(final Configuration configuration) throws IOException, StatoolInfosException { PathCounters result; logger.info("== Probing HttpErrorLog."); String source = configuration.getProbeHttpErrorLogSource(); + String pattern = configuration.getProbeHttpErrorLogPattern(); + String dateTimePattern = configuration.getProbeHttpErrorLogDateTimePattern(); logger.info("source=[{}]", source); + logger.info("pattern=[{}]", pattern); + logger.info("dateTimePattern=[{}]", dateTimePattern); - result = HttpErrorLogAnalyzer.probe(source); + HttpErrorLogs httpErrorLogs = new HttpErrorLogs(FilesUtils.searchByWildcard(source), pattern, dateTimePattern); + + result = HttpErrorLogAnalyzer.probe(httpErrorLogs); // return result; diff --git a/src/fr/devinsy/statoolinfos/metrics/httperrorlog/HttpErrorLog.java b/src/fr/devinsy/statoolinfos/metrics/httperrorlog/HttpErrorLog.java index 05b2b4d..0876c7c 100644 --- a/src/fr/devinsy/statoolinfos/metrics/httperrorlog/HttpErrorLog.java +++ b/src/fr/devinsy/statoolinfos/metrics/httperrorlog/HttpErrorLog.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021-2023 Christian Pierre MOMON + * Copyright (C) 2021-2024 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -43,6 +43,7 @@ public class HttpErrorLog private LocalDateTime time; private String level; + private String message; /** * Instantiates a new http error log. @@ -68,6 +69,11 @@ public class HttpErrorLog return this.level; } + public String getMessage() + { + return this.message; + } + public LocalDateTime getTime() { return this.time; @@ -141,6 +147,11 @@ public class HttpErrorLog this.level = level; } + public void setMessage(final String message) + { + this.message = message; + } + public void setTime(final LocalDateTime time) { this.time = time; @@ -166,4 +177,29 @@ public class HttpErrorLog // return result; } + + /** + * To string log. + * + * @return the string + */ + public String toStringLog() + { + String result; + + // "^(?[a-zA-F0-9\\\\:\\\\.]+) - (?[^\\[]+) + // \\[(?