From 22d3e799493ec398894333ea309b47c87c59ab6e Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Thu, 9 Dec 2021 11:46:35 +0100 Subject: [PATCH] Added metric file update. --- .../statoolinfos/cli/StatoolInfosCLI.java | 3 - .../statoolinfos/core/Configuration.java | 16 +- .../devinsy/statoolinfos/core/LogFilter.java | 67 ------- .../devinsy/statoolinfos/core/ProbeMode.java | 30 +++ .../statoolinfos/metrics/PathCounters.java | 92 +++++++++ .../devinsy/statoolinfos/metrics/Prober.java | 178 +++++++++++++++++- .../metrics/http/HttpErrorLogAnalyzer.java | 4 +- .../statoolinfos/properties/PathProperty.java | 114 ++++++++++- 8 files changed, 428 insertions(+), 76 deletions(-) delete mode 100644 src/fr/devinsy/statoolinfos/core/LogFilter.java create mode 100644 src/fr/devinsy/statoolinfos/core/ProbeMode.java diff --git a/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java b/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java index 6e591ea..545c41b 100644 --- a/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java +++ b/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java @@ -471,8 +471,5 @@ public final class StatoolInfosCLI System.out.println("Bad usage."); displayHelp(); } - - // - logger.info("Done."); } } diff --git a/src/fr/devinsy/statoolinfos/core/Configuration.java b/src/fr/devinsy/statoolinfos/core/Configuration.java index 4a639f6..e058be3 100644 --- a/src/fr/devinsy/statoolinfos/core/Configuration.java +++ b/src/fr/devinsy/statoolinfos/core/Configuration.java @@ -376,6 +376,21 @@ public class Configuration extends PathPropertyList return result; } + /** + * Gets the probe mode. + * + * @return the probe mode + */ + public ProbeMode getProbeMode() + { + ProbeMode result; + + result = ProbeMode.valueOf(StringUtils.upperCase(get("conf.probe.mode"))); + + // + return result; + } + /** * Gets the probe target. * @@ -497,5 +512,4 @@ public class Configuration extends PathPropertyList // return result; } - } diff --git a/src/fr/devinsy/statoolinfos/core/LogFilter.java b/src/fr/devinsy/statoolinfos/core/LogFilter.java deleted file mode 100644 index 9c53f52..0000000 --- a/src/fr/devinsy/statoolinfos/core/LogFilter.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (C) 2020 Christian Pierre MOMON - * - * This file is part of StatoolInfos, simple service statistics tool. - * - * StatoolInfos is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * StatoolInfos is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with StatoolInfos. If not, see . - */ -package fr.devinsy.statoolinfos.core; - -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; - -/** - * The Enum LogFilter. - */ -public enum LogFilter -{ - ALL, - BOT, - NOBOT; - - /** - * Matches. - * - * @param log - * the log - * @return true, if successful - */ - public boolean matches(final HttpAccessLog log) - { - boolean result; - - if (log == null) - { - result = false; - } - else if (this == ALL) - { - result = true; - } - else if ((this == BOT) && (log.isBot())) - { - result = true; - } - else if ((this == NOBOT) && (!log.isBot())) - { - result = true; - } - else - { - result = false; - } - - // - return result; - } -} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/core/ProbeMode.java b/src/fr/devinsy/statoolinfos/core/ProbeMode.java new file mode 100644 index 0000000..6d9bd80 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/core/ProbeMode.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2020-2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.core; + +/** + * The Enum LogFilter. + */ +public enum ProbeMode +{ + FULL, + TODAY, + PREVIOUS_DAY, + PREVIOUS_DAYS_7; +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/metrics/PathCounters.java b/src/fr/devinsy/statoolinfos/metrics/PathCounters.java index 345d88e..cb8d8fa 100644 --- a/src/fr/devinsy/statoolinfos/metrics/PathCounters.java +++ b/src/fr/devinsy/statoolinfos/metrics/PathCounters.java @@ -19,11 +19,13 @@ package fr.devinsy.statoolinfos.metrics; import java.time.LocalDate; +import java.time.YearMonth; import java.time.format.DateTimeFormatter; import java.util.HashMap; import java.util.Locale; import org.apache.commons.lang3.StringUtils; +import org.threeten.extra.YearWeek; import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.strings.StringList; @@ -390,4 +392,94 @@ public class PathCounters extends HashMap put(computeKey(counter.getPath(), counter.getTimeMark()), counter); } } + + /** + * Search by date. + * + * @param day + * the day + * @return the path counters + */ + public PathCounters searchByDate(final LocalDate date) + { + PathCounters result; + + result = new PathCounters(); + + if (date != null) + { + String day = date.toString(); + String week = YearWeek.from(date).toString(); + String month = YearMonth.from(date).toString(); + + for (PathCounter counter : this.values()) + { + TimeMark mark = new TimeMark(counter.getTimeMark()); + + if (mark.isDate() && (counter.getTimeMark().equals(day))) + { + result.put(counter); + } + else if (mark.isYearWeek() && (counter.getTimeMark().equals(week))) + { + result.put(counter); + } + else if (mark.isYearMonth() && (counter.getTimeMark().equals(month))) + { + result.put(counter); + } + } + } + + // + return result; + } + + public PathCounters searchByPeriod(final LocalDate startDate, final LocalDate endDate) + { + PathCounters result; + + result = new PathCounters(); + + if ((startDate != null) && (endDate != null)) + { + if (startDate.isAfter(endDate)) + { + result = searchByPeriod(endDate, startDate); + } + else + { + StringSet days = new StringSet(); + StringSet weeks = new StringSet(); + StringSet months = new StringSet(); + for (LocalDate current = startDate; !current.isAfter(endDate); current = current.plusDays(1)) + { + days.add(current.toString()); + weeks.add(YearWeek.from(current).toString()); + months.add(YearMonth.from(current).toString()); + } + + for (PathCounter counter : this.values()) + { + TimeMark mark = new TimeMark(counter.getTimeMark()); + + if (mark.isDate() && (days.contains(counter.getTimeMark()))) + { + result.put(counter); + } + else if (mark.isYearWeek() && (weeks.contains(counter.getTimeMark()))) + { + result.put(counter); + } + else if (mark.isYearMonth() && (months.contains(counter.getTimeMark()))) + { + result.put(counter); + } + } + } + } + + // + return result; + } } diff --git a/src/fr/devinsy/statoolinfos/metrics/Prober.java b/src/fr/devinsy/statoolinfos/metrics/Prober.java index f2c65ed..ab43ded 100644 --- a/src/fr/devinsy/statoolinfos/metrics/Prober.java +++ b/src/fr/devinsy/statoolinfos/metrics/Prober.java @@ -22,17 +22,27 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import fr.devinsy.statoolinfos.core.Configuration; import fr.devinsy.statoolinfos.core.Factory; +import fr.devinsy.statoolinfos.core.ProbeMode; import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer; import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer; +import fr.devinsy.statoolinfos.properties.PathProperties; +import fr.devinsy.statoolinfos.properties.PathProperty; +import fr.devinsy.statoolinfos.properties.PathPropertyUtils; import fr.devinsy.strings.StringList; import fr.devinsy.strings.StringsUtils; @@ -45,6 +55,8 @@ public class Prober public static final String DEFAULT_CHARSET_NAME = "UTF-8"; + public static final Pattern YEAR_PATTERN = Pattern.compile("^\\d{4}$"); + /** * Instantiates a new prober. */ @@ -52,6 +64,23 @@ public class Prober { } + /** + * Extract year. + * + * @param line + * the line + * @return the string + */ + private static boolean isYear(final String value) + { + boolean result; + + result = YEAR_PATTERN.matcher(value).matches(); + + // + return result; + } + /** * Probe. * @@ -106,7 +135,35 @@ public class Prober } // - logger.info("== Writing."); + if (types.containsAnyIgnoreCase("Mumble")) + { + logger.info("== Processing Mumble."); + String source = configuration.getProbeHttpErrorLogSource(); + logger.info("source=[{}]", source); + + // PathCounters data = HttpErrorLogAnalyzer.probe(source); + // counters.putAll(data); + } + + // Filter. + logger.info("== Filtering"); + if (configuration.getProbeMode() == ProbeMode.TODAY) + { + logger.info("=== TODAY"); + counters = counters.searchByPeriod(LocalDate.now(), LocalDate.now()); + } + else if (configuration.getProbeMode() == ProbeMode.PREVIOUS_DAY) + { + logger.info("=== PREVIOUS_DAY"); + counters = counters.searchByPeriod(LocalDate.now().minusDays(1), LocalDate.now()); + } + else if (configuration.getProbeMode() == ProbeMode.PREVIOUS_DAYS_7) + { + logger.info("=== PREVIOUS_WEEK"); + counters = counters.searchByPeriod(LocalDate.now().minusDays(7), LocalDate.now()); + } + + // File target = configuration.getProbeTarget(); logger.info("target=[{}]", target); @@ -116,10 +173,38 @@ public class Prober } else { + if (target.exists()) + { + logger.info("ModeProbe=" + configuration.getProbeMode()); + if (configuration.getProbeMode() != ProbeMode.FULL) + { + // Load. + logger.info("== Reading previous target file."); + PathCounters previousCounters = readMetrics(target); + logger.info("previous size={}", previousCounters.size()); + + // Merge. + logger.info("== Merging"); + for (PathCounter counter : counters.values()) + { + PathCounter previousCounter = previousCounters.get(counter.getPath(), counter.getTimeMark()); + previousCounter.setCounter(counter.getCounter()); + } + + // + counters = previousCounters; + } + + // + logger.info("== Backing previous target file."); + target.renameTo(new File(target.getParentFile(), target.getName() + ".bak")); + } + writeMetrics(target, counters); } // + logger.info("== Writing."); logger.info("size={}", counters.size()); } @@ -140,6 +225,97 @@ public class Prober probe(configuration); } + /** + * Read metrics. + * + * @param inputFile + * the input file + * @return the path counters + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public static PathCounters readMetrics(final File inputFile) throws IOException + { + PathCounters result; + + result = new PathCounters(); + + PathProperties lines = PathPropertyUtils.load(inputFile); + + for (PathProperty property : lines) + { + String metricName = property.getMetricName(); + if (metricName != null) + { + String leaf = property.getLeaf(); + if (isYear(leaf)) + { + PathCounter counter = new PathCounter(metricName, leaf); + counter.setCounter(NumberUtils.createLong(property.getValue())); + result.put(counter); + } + else if (StringUtils.equals(leaf, "months")) + { + String year = property.getMetricYear(); + + StringList values = StatoolInfosUtils.splitMonthValues(property.getValue()); + int monthIndex = 1; + for (String value : values) + { + if (!StringUtils.isBlank(value)) + { + PathCounter counter = new PathCounter(metricName, String.format("%s-%02d", year, monthIndex)); + counter.setCounter(NumberUtils.createLong(value)); + result.put(counter); + } + + monthIndex += 1; + } + } + else if (StringUtils.equals(leaf, "weeks")) + { + String year = property.getMetricYear(); + + StringList values = StatoolInfosUtils.splitWeekValues(property.getValue()); + int weekIndex = 1; + for (String value : values) + { + if (!StringUtils.isBlank(value)) + { + PathCounter counter = new PathCounter(metricName, String.format("%s-W%02d", year, weekIndex)); + counter.setCounter(NumberUtils.createLong(value)); + result.put(counter); + } + + weekIndex += 1; + } + } + else if (StringUtils.equals(leaf, "days")) + { + String year = property.getMetricYear(); + + StringList values = StatoolInfosUtils.splitDayValues(property.getValue()); + LocalDate day = LocalDate.of(Integer.parseInt(year), 1, 1); + for (String value : values) + { + if (!StringUtils.isBlank(value)) + { + String date = day.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE)); + PathCounter counter = new PathCounter(metricName, date); + counter.setCounter(NumberUtils.createLong(value)); + result.put(counter); + } + + day = day.plusDays(1); + } + } + } + } + + // + return result; + } + /** * Write metrics. * diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java index 0f2eff5..ce5595d 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java @@ -44,7 +44,7 @@ public class HttpErrorLogAnalyzer public static final String DEFAULT_CHARSET_NAME = "UTF-8"; public static final Pattern NGINX_ERROR_PATTERN = Pattern.compile("^(?