diff --git a/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java b/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java index 2c33aac..a307840 100644 --- a/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java +++ b/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2021 Christian Pierre MOMON + * Copyright (C) 2020-2022 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -197,6 +197,30 @@ public final class StatoolInfosCLI return result; } + /** + * Normalize bash parameter. + * + * @param value + * the value + * @return the string + */ + private static String normalizeBashParameter(final String value) + { + String result; + + if (value == null) + { + result = null; + } + else + { + result = StringUtils.trim(value).replace("\\*", "*").replace("\\?", "?"); + } + + // + return result; + } + /** * Parses the log filter option. * @@ -508,6 +532,23 @@ public final class StatoolInfosCLI } System.out.println(chrono.format()); } + else if (isMatching(args, "testHttpAccessLog", "\\s*\\S+\\s*")) + { + System.out.println("param1=" + args[1]); + String source = normalizeBashParameter(args[1]); + System.out.println("source=" + source); + Files files = FilesUtils.searchByWildcard(source); + System.out.println(files.size()); + try + { + StatoolInfos.testHttpAccessLog(files); + } + catch (Exception exception) + { + logger.error("Error: {}", exception.getMessage()); + exception.printStackTrace(); + } + } else { System.out.println("Bad usage."); diff --git a/src/fr/devinsy/statoolinfos/core/StatoolInfos.java b/src/fr/devinsy/statoolinfos/core/StatoolInfos.java index a68b296..abfa81c 100644 --- a/src/fr/devinsy/statoolinfos/core/StatoolInfos.java +++ b/src/fr/devinsy/statoolinfos/core/StatoolInfos.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 Christian Pierre MOMON + * Copyright (C) 2020-2022 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -34,7 +34,8 @@ import fr.devinsy.statoolinfos.crawl.Crawler; import fr.devinsy.statoolinfos.htmlize.Htmlizer; import fr.devinsy.statoolinfos.metrics.Prober; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogParser; import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathPropertyUtils; import fr.devinsy.statoolinfos.stats.ip.IpStat; @@ -46,6 +47,7 @@ import fr.devinsy.statoolinfos.stats.visitor.VisitorStator; import fr.devinsy.statoolinfos.uptime.UptimeJournal; import fr.devinsy.statoolinfos.uptime.UptimeSurveyor; import fr.devinsy.statoolinfos.util.Chrono; +import fr.devinsy.statoolinfos.util.Files; import fr.devinsy.statoolinfos.util.LineIterator; import fr.devinsy.strings.StringList; import fr.devinsy.strings.StringsUtils; @@ -180,7 +182,7 @@ public class StatoolInfos while (iterator.hasNext()) { String line = iterator.next(); - HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN); + HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); if (filter.matches(log)) { @@ -230,7 +232,7 @@ public class StatoolInfos while (iterator.hasNext()) { String line = iterator.next(); - HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN); + HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); if (filter.matches(log)) { @@ -281,7 +283,7 @@ public class StatoolInfos while (iterator.hasNext()) { String line = iterator.next(); - HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN); + HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); if (filter.matches(log)) { @@ -347,7 +349,7 @@ public class StatoolInfos while (iterator.hasNext()) { String line = iterator.next(); - HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN); + HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); if (filter.matches(log)) { stator.putLog(log); @@ -398,7 +400,7 @@ public class StatoolInfos while (iterator.hasNext()) { String line = iterator.next(); - HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN); + HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); if (filter.matches(log)) { stator.putLog(log); @@ -449,7 +451,7 @@ public class StatoolInfos while (iterator.hasNext()) { String line = iterator.next(); - HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN); + HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); if (filter.matches(log)) { stator.putLog(log); @@ -531,6 +533,22 @@ public class StatoolInfos } } + /** + * Test http access log. + * + * @param source + * the source + * @throws IOException + */ + public static void testHttpAccessLog(final Files source) throws IOException + { + HttpAccessLogIterator logs = new HttpAccessLogIterator(source); + while (logs.hasNext()) + { + System.out.println(logs.next().toStringLog()); + } + } + /** * Uptime. * diff --git a/src/fr/devinsy/statoolinfos/metrics/Prober.java b/src/fr/devinsy/statoolinfos/metrics/Prober.java index ae2cffd..c9cbadd 100644 --- a/src/fr/devinsy/statoolinfos/metrics/Prober.java +++ b/src/fr/devinsy/statoolinfos/metrics/Prober.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2021 Christian Pierre MOMON + * Copyright (C) 2020-2022 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -40,6 +40,7 @@ import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer; import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer; +import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber; import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber; import fr.devinsy.statoolinfos.metrics.mumble.MumbleProber; import fr.devinsy.statoolinfos.metrics.privatebin.PrivatebinProber; @@ -134,13 +135,31 @@ public class Prober if (types.containsAnyIgnoreCase("Framadate")) { logger.info("== Processing Framadate."); - String source = configuration.getProbeHttpErrorLogSource(); + String source = configuration.getProbeHttpAccessLogSource(); logger.info("source=[{}]", source); // PathCounters data = HttpErrorLogAnalyzer.probe(source); // counters.putAll(data); } + // + if (types.containsAnyIgnoreCase("LibreQR")) + { + logger.info("== Processing LibreQR."); + + String source = configuration.getProbeHttpAccessLogSource(); + logger.info("source=[{}]", source); + + String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); + logger.info("pattern=[{}]", httpAccessLogRegex); + + File dataDirectory = configuration.getAsFile("conf.probe.libreqr.data"); + logger.info("dataDirectory=[{}]", dataDirectory); + + PathCounters data = LibreQRProber.probe(source, httpAccessLogRegex, dataDirectory); + counters.putAll(data); + } + // if (types.containsAnyIgnoreCase("Minetest")) { @@ -168,10 +187,10 @@ public class Prober { logger.info("== Processing Privatebin."); String httpAccessLogs = configuration.getProbeHttpAccessLogSource(); - String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); - File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data"); logger.info("source=[{}]", httpAccessLogs); + String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); logger.info("pattern=[{}]", httpAccessLogRegex); + File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data"); logger.info("dataDirectory=[{}]", dataDirectory); PathCounters data = PrivatebinProber.probe(httpAccessLogs, httpAccessLogRegex, dataDirectory); diff --git a/src/fr/devinsy/statoolinfos/metrics/TimeMarkUtils.java b/src/fr/devinsy/statoolinfos/metrics/TimeMarkUtils.java index cc968ba..298c6fa 100644 --- a/src/fr/devinsy/statoolinfos/metrics/TimeMarkUtils.java +++ b/src/fr/devinsy/statoolinfos/metrics/TimeMarkUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Christian Pierre MOMON + * Copyright (C) 2022 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -21,6 +21,7 @@ package fr.devinsy.statoolinfos.metrics; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.YearMonth; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.regex.Pattern; @@ -218,6 +219,23 @@ public class TimeMarkUtils return result; } + /** + * Year month of. + * + * @param date + * the date + * @return the string + */ + public static String yearMonthOf(final ZonedDateTime date) + { + String result; + + result = date.format(DateTimeFormatter.ofPattern("yyyy-MM", Locale.FRANCE)); + + // + return result; + } + /** * Year of. * @@ -235,6 +253,23 @@ public class TimeMarkUtils return result; } + /** + * Year of. + * + * @param date + * the date + * @return the string + */ + public static String yearOf(final ZonedDateTime date) + { + String result; + + result = date.format(DateTimeFormatter.ofPattern("yyyy", Locale.FRANCE)); + + // + return result; + } + /** * Year week of. * @@ -251,4 +286,21 @@ public class TimeMarkUtils // return result; } + + /** + * Year week of. + * + * @param date + * the date + * @return the string + */ + public static String yearWeekOf(final ZonedDateTime date) + { + String result; + + result = YearWeek.from(date).toString(); + + // + return result; + } } diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java index ab3ee16..d56d13f 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021 Christian Pierre MOMON + * Copyright (C) 2021-2022 Christian Pierre MOMON * * This file is part of StatoolInfos, simple service statistics tool. * @@ -19,6 +19,8 @@ package fr.devinsy.statoolinfos.metrics.http; import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Locale; import java.util.regex.Pattern; @@ -40,7 +42,7 @@ public class HttpAccessLog private String ip; private String remoteUser; - private LocalDateTime time; + private ZonedDateTime time; private String request; private HttpStatus status; private long bodyBytesSent; @@ -134,7 +136,7 @@ public class HttpAccessLog public LocalDateTime getTime() { - return this.time; + return this.time.toLocalDateTime(); } public UserAgent getUserAgent() @@ -317,7 +319,18 @@ public class HttpAccessLog this.status = status; } + /** + * Sets the time. + * + * @param time + * the new time + */ public void setTime(final LocalDateTime time) + { + this.time = ZonedDateTime.of(time, ZoneId.systemDefault()); + } + + public void setTime(final ZonedDateTime time) { this.time = time; } @@ -353,4 +366,34 @@ public class HttpAccessLog // return result; } + + /** + * To string log. + * + * @return the string + */ + public String toStringLog() + { + String result; + + // "^(?[a-zA-F0-9\\\\:\\\\.]+) - (?[^\\[]+) + // \\[(?