statool/statoolinfosweb
statool/statoolinfosweb
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Activity

Added metric file update.

Browse source
This commit is contained in:
Christian P. MOMON 2021-12-09 11:46:35 +01:00
parent 8a563ff194
commit 22d3e79949
8 changed files with 428 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java
View file

@ -471,8 +471,5 @@ public final class StatoolInfosCLI
System.out.println("Bad usage.");
displayHelp();
}
//
logger.info("Done.");
}
}

16
src/fr/devinsy/statoolinfos/core/Configuration.java
View file

@ -376,6 +376,21 @@ public class Configuration extends PathPropertyList
return result;
}
/**
* Gets the probe mode.
*
* @return the probe mode
*/
public ProbeMode getProbeMode()
{
ProbeMode result;
result = ProbeMode.valueOf(StringUtils.upperCase(get("conf.probe.mode")));
//
return result;
}
/**
* Gets the probe target.
*
@ -497,5 +512,4 @@ public class Configuration extends PathPropertyList
//
return result;
}
}

67
src/fr/devinsy/statoolinfos/core/LogFilter.java
View file

@ -1,67 +0,0 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.core;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
/**
* The Enum LogFilter.
*/
public enum LogFilter
{
ALL,
BOT,
NOBOT;
/**
* Matches.
*
* @param log
* the log
* @return true, if successful
*/
public boolean matches(final HttpAccessLog log)
{
boolean result;
if (log == null)
{
result = false;
}
else if (this == ALL)
{
result = true;
}
else if ((this == BOT) && (log.isBot()))
{
result = true;
}
else if ((this == NOBOT) && (!log.isBot()))
{
result = true;
}
else
{
result = false;
}
//
return result;
}
}

30
src/fr/devinsy/statoolinfos/core/ProbeMode.java Normal file
View file

@ -0,0 +1,30 @@
/*
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.core;
/**
* The Enum LogFilter.
*/
public enum ProbeMode
{
FULL,
TODAY,
PREVIOUS_DAY,
PREVIOUS_DAYS_7;
}

92
src/fr/devinsy/statoolinfos/metrics/PathCounters.java
View file

@ -19,11 +19,13 @@
package fr.devinsy.statoolinfos.metrics;
import java.time.LocalDate;
import java.time.YearMonth;
import java.time.format.DateTimeFormatter;
import java.util.HashMap;
import java.util.Locale;
import org.apache.commons.lang3.StringUtils;
import org.threeten.extra.YearWeek;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.strings.StringList;
@ -390,4 +392,94 @@ public class PathCounters extends HashMap<String, PathCounter>
put(computeKey(counter.getPath(), counter.getTimeMark()), counter);
}
}
/**
* Search by date.
*
* @param day
* the day
* @return the path counters
*/
public PathCounters searchByDate(final LocalDate date)
{
PathCounters result;
result = new PathCounters();
if (date != null)
{
String day = date.toString();
String week = YearWeek.from(date).toString();
String month = YearMonth.from(date).toString();
for (PathCounter counter : this.values())
{
TimeMark mark = new TimeMark(counter.getTimeMark());
if (mark.isDate() && (counter.getTimeMark().equals(day)))
{
result.put(counter);
}
else if (mark.isYearWeek() && (counter.getTimeMark().equals(week)))
{
result.put(counter);
}
else if (mark.isYearMonth() && (counter.getTimeMark().equals(month)))
{
result.put(counter);
}
}
}
//
return result;
}
public PathCounters searchByPeriod(final LocalDate startDate, final LocalDate endDate)
{
PathCounters result;
result = new PathCounters();
if ((startDate != null) && (endDate != null))
{
if (startDate.isAfter(endDate))
{
result = searchByPeriod(endDate, startDate);
}
else
{
StringSet days = new StringSet();
StringSet weeks = new StringSet();
StringSet months = new StringSet();
for (LocalDate current = startDate; !current.isAfter(endDate); current = current.plusDays(1))
{
days.add(current.toString());
weeks.add(YearWeek.from(current).toString());
months.add(YearMonth.from(current).toString());
}
for (PathCounter counter : this.values())
{
TimeMark mark = new TimeMark(counter.getTimeMark());
if (mark.isDate() && (days.contains(counter.getTimeMark())))
{
result.put(counter);
}
else if (mark.isYearWeek() && (weeks.contains(counter.getTimeMark())))
{
result.put(counter);
}
else if (mark.isYearMonth() && (months.contains(counter.getTimeMark())))
{
result.put(counter);
}
}
}
}
//
return result;
}
}

178
src/fr/devinsy/statoolinfos/metrics/Prober.java
View file

@ -22,17 +22,27 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.Configuration;
import fr.devinsy.statoolinfos.core.Factory;
import fr.devinsy.statoolinfos.core.ProbeMode;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer;
import fr.devinsy.statoolinfos.properties.PathProperties;
import fr.devinsy.statoolinfos.properties.PathProperty;
import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
import fr.devinsy.strings.StringList;
import fr.devinsy.strings.StringsUtils;
@ -45,6 +55,8 @@ public class Prober
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
public static final Pattern YEAR_PATTERN = Pattern.compile("^\\d{4}$");
/**
* Instantiates a new prober.
*/
@ -52,6 +64,23 @@ public class Prober
{
}
/**
* Extract year.
*
* @param line
* the line
* @return the string
*/
private static boolean isYear(final String value)
{
boolean result;
result = YEAR_PATTERN.matcher(value).matches();
//
return result;
}
/**
* Probe.
*
@ -106,7 +135,35 @@ public class Prober
}
//
logger.info("== Writing.");
if (types.containsAnyIgnoreCase("Mumble"))
{
logger.info("== Processing Mumble.");
String source = configuration.getProbeHttpErrorLogSource();
logger.info("source=[{}]", source);
// PathCounters data = HttpErrorLogAnalyzer.probe(source);
// counters.putAll(data);
}
// Filter.
logger.info("== Filtering");
if (configuration.getProbeMode() == ProbeMode.TODAY)
{
logger.info("=== TODAY");
counters = counters.searchByPeriod(LocalDate.now(), LocalDate.now());
}
else if (configuration.getProbeMode() == ProbeMode.PREVIOUS_DAY)
{
logger.info("=== PREVIOUS_DAY");
counters = counters.searchByPeriod(LocalDate.now().minusDays(1), LocalDate.now());
}
else if (configuration.getProbeMode() == ProbeMode.PREVIOUS_DAYS_7)
{
logger.info("=== PREVIOUS_WEEK");
counters = counters.searchByPeriod(LocalDate.now().minusDays(7), LocalDate.now());
}
//
File target = configuration.getProbeTarget();
logger.info("target=[{}]", target);
@ -116,10 +173,38 @@ public class Prober
}
else
{
if (target.exists())
{
logger.info("ModeProbe=" + configuration.getProbeMode());
if (configuration.getProbeMode() != ProbeMode.FULL)
{
// Load.
logger.info("== Reading previous target file.");
PathCounters previousCounters = readMetrics(target);
logger.info("previous size={}", previousCounters.size());
// Merge.
logger.info("== Merging");
for (PathCounter counter : counters.values())
{
PathCounter previousCounter = previousCounters.get(counter.getPath(), counter.getTimeMark());
previousCounter.setCounter(counter.getCounter());
}
//
counters = previousCounters;
}
//
logger.info("== Backing previous target file.");
target.renameTo(new File(target.getParentFile(), target.getName() + ".bak"));
}
writeMetrics(target, counters);
}
//
logger.info("== Writing.");
logger.info("size={}", counters.size());
}
@ -140,6 +225,97 @@ public class Prober
probe(configuration);
}
/**
* Read metrics.
*
* @param inputFile
* the input file
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static PathCounters readMetrics(final File inputFile) throws IOException
{
PathCounters result;
result = new PathCounters();
PathProperties lines = PathPropertyUtils.load(inputFile);
for (PathProperty property : lines)
{
String metricName = property.getMetricName();
if (metricName != null)
{
String leaf = property.getLeaf();
if (isYear(leaf))
{
PathCounter counter = new PathCounter(metricName, leaf);
counter.setCounter(NumberUtils.createLong(property.getValue()));
result.put(counter);
}
else if (StringUtils.equals(leaf, "months"))
{
String year = property.getMetricYear();
StringList values = StatoolInfosUtils.splitMonthValues(property.getValue());
int monthIndex = 1;
for (String value : values)
{
if (!StringUtils.isBlank(value))
{
PathCounter counter = new PathCounter(metricName, String.format("%s-%02d", year, monthIndex));
counter.setCounter(NumberUtils.createLong(value));
result.put(counter);
}
monthIndex += 1;
}
}
else if (StringUtils.equals(leaf, "weeks"))
{
String year = property.getMetricYear();
StringList values = StatoolInfosUtils.splitWeekValues(property.getValue());
int weekIndex = 1;
for (String value : values)
{
if (!StringUtils.isBlank(value))
{
PathCounter counter = new PathCounter(metricName, String.format("%s-W%02d", year, weekIndex));
counter.setCounter(NumberUtils.createLong(value));
result.put(counter);
}
weekIndex += 1;
}
}
else if (StringUtils.equals(leaf, "days"))
{
String year = property.getMetricYear();
StringList values = StatoolInfosUtils.splitDayValues(property.getValue());
LocalDate day = LocalDate.of(Integer.parseInt(year), 1, 1);
for (String value : values)
{
if (!StringUtils.isBlank(value))
{
String date = day.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
PathCounter counter = new PathCounter(metricName, date);
counter.setCounter(NumberUtils.createLong(value));
result.put(counter);
}
day = day.plusDays(1);
}
}
}
}
//
return result;
}
/**
* Write metrics.
*

4
src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java
View file

@ -44,7 +44,7 @@ public class HttpErrorLogAnalyzer
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
public static final Pattern NGINX_ERROR_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
public static final Pattern APACHE_ERROR_PATTERN = Pattern.compile("TODO^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
public static final Pattern APACHE_ERROR_PATTERN = Pattern.compile("^\\[(?<time>[^\\]]+)\\]\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
private PathCounters counters;
private int errorCount;
@ -169,7 +169,7 @@ public class HttpErrorLogAnalyzer
if (matcher.matches())
{
result = new HttpErrorLog();
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)));
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss.SSSSSS yyyy").withLocale(Locale.ENGLISH)));
result.setLevel(matcher.group("level"));
}
else

114
src/fr/devinsy/statoolinfos/properties/PathProperty.java
View file

@ -18,6 +18,9 @@
*/
package fr.devinsy.statoolinfos.properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
/**
@ -25,6 +28,10 @@ import org.apache.commons.lang3.StringUtils;
*/
public class PathProperty
{
public static final Pattern METRIC_LABEL_PATTERN = Pattern.compile("^metrics\\.(?<label>\\S+)\\.(name|description|\\d{4}(\\.(months|weeks|days))?)$");
public static final Pattern METRIC_NAME_PATTERN = Pattern.compile("^(?<name>metrics\\.\\S+)\\.(name|description|\\d{4}(\\.(months|weeks|days))?)$");
public static final Pattern METRIC_YEAR_PATTERN = Pattern.compile("^metrics\\.\\S+\\.(?<year>\\d{4})(\\.(months|weeks|days))?$");
private String path;
private String value;
@ -64,6 +71,101 @@ public class PathProperty
return result;
}
/**
* Gets the metric name.
*
* <pre>
* ("service.name") = null
* ("metrics.https.visitors.name") = "https.visitors"
* ("metrics.https.visitors.2020") = "https.visitors"
* ("metrics.https.visitors.2020.months") = "https.visitors"
* </pre>
*
* @return the metric name
*/
public String getMetricLabel()
{
String result;
Matcher matcher = METRIC_LABEL_PATTERN.matcher(this.path);
if (matcher.matches())
{
result = matcher.group("label");
}
else
{
result = null;
}
//
return result;
}
/**
* Gets the metric name.
*
* <pre>
* ("service.name") = null
* ("metrics.https.visitors.name") = "metrics.https.visitors"
* ("metrics.https.visitors.2020") = "metrics.https.visitors"
* ("metrics.https.visitors.2020.months") = "metrics.https.visitors"
* </pre>
*
* @return the metric name
*/
public String getMetricName()
{
String result;
Matcher matcher = METRIC_NAME_PATTERN.matcher(this.path);
if (matcher.matches())
{
result = matcher.group("name");
}
else
{
result = null;
}
//
return result;
}
/**
* Gets the metric year.
*
* <pre>
* ("service.name") = null
* ("metrics.https.visitors.name") = null
* ("metrics.https.visitors.2020") = "2020"
* ("metrics.https.visitors.2020.months") = "2020"
* </pre>
*
* @return the metric name
*/
public String getMetricYear()
{
String result;
Matcher matcher = METRIC_YEAR_PATTERN.matcher(this.path);
if (matcher.matches())
{
result = matcher.group("year");
}
else
{
result = null;
}
//
return result;
}
/**
* Gets the path.
*
* @return the path
*/
public String getPath()
{
return this.path;
@ -74,11 +176,19 @@ public class PathProperty
*
* @return the prefix
*/
private String getPrefix()
public String getPrefix()
{
String result;
result = this.path.substring(0, this.path.indexOf("."));
int index = this.path.indexOf(".");
if (index == -1)
{
result = null;
}
else
{
result = this.path.substring(0, index);
}
//
return result;