Refactored http error log metrics. Added list errfile and list errlog
commands.
This commit is contained in:
parent
633246dddd
commit
d453979eef
11 changed files with 1091 additions and 163 deletions
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
|
||||
* Copyright (C) 2020-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
|
|
@ -124,6 +124,8 @@ public final class StatoolInfosCLI
|
|||
message.appendln(" statoolinfos stat ip [-bot|-nobot] <logfilesorconfigfile> generate stats about ip from http log file");
|
||||
message.appendln(" statoolinfos stat ua [-bot|-nobot] <logfilesorconfigfile> generate stats about user agent from http log file");
|
||||
message.appendln(" statoolinfos stat visitor [-bot|-nobot] <logfilesorconfigfile> generate stats about visitor (ip+ua) from http log file");
|
||||
message.appendln(" statoolinfos list errfile <logfilesorconfigfile> display http error log files");
|
||||
message.appendln(" statoolinfos list errlog <logfilesorconfigfile> display http error log lines");
|
||||
|
||||
System.out.print(message.toString());
|
||||
}
|
||||
|
|
@ -351,7 +353,7 @@ public final class StatoolInfosCLI
|
|||
{
|
||||
File configurationFile = new File(StringUtils.trim(args[2]));
|
||||
|
||||
StatoolInfos.listFiles(configurationFile);
|
||||
StatoolInfos.listAccessFiles(configurationFile);
|
||||
}
|
||||
else if (CLIUtils.isMatchingEllipsis(args, "list", "(file|files)", ".+"))
|
||||
{
|
||||
|
|
@ -361,7 +363,23 @@ public final class StatoolInfosCLI
|
|||
source.add(new File(args[index]));
|
||||
}
|
||||
|
||||
StatoolInfos.listFiles(source);
|
||||
StatoolInfos.listAccessFiles(source);
|
||||
}
|
||||
else if (CLIUtils.isMatching(args, "list", "(errfile|errfiles)", ".+\\.conf"))
|
||||
{
|
||||
File configurationFile = new File(StringUtils.trim(args[2]));
|
||||
|
||||
StatoolInfos.listErrorFiles(configurationFile);
|
||||
}
|
||||
else if (CLIUtils.isMatchingEllipsis(args, "list", "(errfile|errfiles)", ".+"))
|
||||
{
|
||||
Files source = new Files();
|
||||
for (int index = 2; index < args.length; index++)
|
||||
{
|
||||
source.add(new File(args[index]));
|
||||
}
|
||||
|
||||
StatoolInfos.listErrorFiles(source);
|
||||
}
|
||||
else if (CLIUtils.isMatching(args, "list", "(log|logs)", "(-all|-bot|-nobot)", ".+\\.conf*"))
|
||||
{
|
||||
|
|
@ -397,6 +415,22 @@ public final class StatoolInfosCLI
|
|||
|
||||
StatoolInfos.listLogs(source, BotFilter.ALL);
|
||||
}
|
||||
else if (CLIUtils.isMatching(args, "list", "(errlog|errlogs)", ".+\\.conf"))
|
||||
{
|
||||
File configurationFile = new File(StringUtils.trim(args[2]));
|
||||
|
||||
StatoolInfos.listErrorLogs(configurationFile);
|
||||
}
|
||||
else if (CLIUtils.isMatchingEllipsis(args, "list", "(errlog|errlogs)", ".+"))
|
||||
{
|
||||
Files source = new Files();
|
||||
for (int index = 2; index < args.length; index++)
|
||||
{
|
||||
source.add(new File(args[index]));
|
||||
}
|
||||
|
||||
StatoolInfos.listErrorLogs(source);
|
||||
}
|
||||
else if (CLIUtils.isMatching(args, "list", "(useragent|ua)", "(-all|-bot|-nobot)", ".+\\.conf*"))
|
||||
{
|
||||
BotFilter filter = parseLogFilterOption(args[2]);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
|
||||
* Copyright (C) 2020-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
|
|
@ -25,6 +25,7 @@ import java.net.URL;
|
|||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import fr.devinsy.statoolinfos.properties.PathProperties;
|
||||
import fr.devinsy.statoolinfos.properties.PathProperty;
|
||||
import fr.devinsy.statoolinfos.properties.PathPropertyList;
|
||||
import fr.devinsy.strings.StringList;
|
||||
|
||||
|
|
@ -54,6 +55,23 @@ public class Configuration extends PathPropertyList
|
|||
super(properties);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the.
|
||||
*
|
||||
* @param path
|
||||
* the path
|
||||
* @param value
|
||||
* the value
|
||||
*/
|
||||
public void add(final String path, final String value)
|
||||
{
|
||||
if (!StringUtils.isBlank(path))
|
||||
{
|
||||
PathProperty property = new PathProperty(path, value);
|
||||
this.add(property);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the builds the directory.
|
||||
*
|
||||
|
|
@ -443,6 +461,36 @@ public class Configuration extends PathPropertyList
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the probe http error log date time pattern.
|
||||
*
|
||||
* @return the probe http error log date time pattern
|
||||
*/
|
||||
public String getProbeHttpErrorLogDateTimePattern()
|
||||
{
|
||||
String result;
|
||||
|
||||
result = get("conf.probe.httperrorlog.datetimepattern");
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the probe http error log pattern.
|
||||
*
|
||||
* @return the probe http error log pattern
|
||||
*/
|
||||
public String getProbeHttpErrorLogPattern()
|
||||
{
|
||||
String result;
|
||||
|
||||
result = get("conf.probe.httperroorlog.pattern");
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the probe http error log source.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
|
||||
* Copyright (C) 2020-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
|
|
@ -33,8 +33,10 @@ import fr.devinsy.statoolinfos.build.Builder;
|
|||
import fr.devinsy.statoolinfos.crawl.Crawler;
|
||||
import fr.devinsy.statoolinfos.htmlize.Htmlizer;
|
||||
import fr.devinsy.statoolinfos.metrics.Prober;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
|
||||
import fr.devinsy.statoolinfos.metrics.httpaccess.HttpAccessLog;
|
||||
import fr.devinsy.statoolinfos.metrics.httpaccess.HttpAccessLogs;
|
||||
import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLog;
|
||||
import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogs;
|
||||
import fr.devinsy.statoolinfos.properties.PathProperties;
|
||||
import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
|
||||
import fr.devinsy.statoolinfos.stats.ip.IpStat;
|
||||
|
|
@ -169,7 +171,7 @@ public class StatoolInfos
|
|||
* @param configurationFile
|
||||
* the configuration file
|
||||
*/
|
||||
public static void listFiles(final File configurationFile)
|
||||
public static void listAccessFiles(final File configurationFile)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
|
@ -209,7 +211,7 @@ public class StatoolInfos
|
|||
* @param files
|
||||
* the files
|
||||
*/
|
||||
public static void listFiles(final Files files)
|
||||
public static void listAccessFiles(final Files files)
|
||||
{
|
||||
if (files != null)
|
||||
{
|
||||
|
|
@ -223,6 +225,140 @@ public class StatoolInfos
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List error files.
|
||||
*
|
||||
* @param configurationFile
|
||||
* the configuration file
|
||||
*/
|
||||
public static void listErrorFiles(final File configurationFile)
|
||||
{
|
||||
try
|
||||
{
|
||||
if ((configurationFile == null) || (!configurationFile.exists()))
|
||||
{
|
||||
System.out.println("No configuration file found.");
|
||||
}
|
||||
else
|
||||
{
|
||||
System.out.println("Listing HttpErrorLog files from [" + configurationFile.toString() + "]");
|
||||
Configuration configuration = Factory.loadConfiguration(configurationFile);
|
||||
|
||||
logger.info("== List HttpErrorLog files.");
|
||||
String source = configuration.getProbeHttpErrorLogSource();
|
||||
logger.info("source=[{}]", source);
|
||||
|
||||
Files files = FilesUtils.searchByWildcard(source);
|
||||
for (File file : files)
|
||||
{
|
||||
System.out.println(file.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (StatoolInfosException exception)
|
||||
{
|
||||
exception.printStackTrace();
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List error files.
|
||||
*
|
||||
* @param files
|
||||
* the files
|
||||
*/
|
||||
public static void listErrorFiles(final Files files)
|
||||
{
|
||||
if (files != null)
|
||||
{
|
||||
System.out.println("Listing HttpErrorLog files from [" + files.toString() + "]");
|
||||
|
||||
logger.info("== List HttpErrorLog files");
|
||||
for (File file : files)
|
||||
{
|
||||
System.out.println(file.getAbsolutePath());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List error logs.
|
||||
*
|
||||
* @param configurationFile
|
||||
* the configuration file
|
||||
*/
|
||||
public static void listErrorLogs(final File configurationFile)
|
||||
{
|
||||
try
|
||||
{
|
||||
if ((configurationFile == null) || (!configurationFile.exists()))
|
||||
{
|
||||
System.out.println("No configuration file found.");
|
||||
}
|
||||
else
|
||||
{
|
||||
System.out.println("Testing HttpErrorLog lines from [" + configurationFile.toString() + "]");
|
||||
Configuration configuration = Factory.loadConfiguration(configurationFile);
|
||||
|
||||
logger.info("== Testing HttpErrorLog lines.");
|
||||
String source = configuration.getProbeHttpErrorLogSource();
|
||||
String dateTimePattern = configuration.getProbeHttpErrorLogDateTimePattern();
|
||||
String pattern = configuration.getProbeHttpErrorLogPattern();
|
||||
logger.info("source=[{}]", source);
|
||||
logger.info("pattern=[{}]", pattern);
|
||||
logger.info("dateTimePattern=[{}]", dateTimePattern);
|
||||
|
||||
HttpErrorLogs logs = new HttpErrorLogs(FilesUtils.searchByWildcard(source), pattern, dateTimePattern);
|
||||
|
||||
for (HttpErrorLog log : logs)
|
||||
{
|
||||
System.out.println(log.toStringLog());
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.error("Error: {}", exception.getMessage());
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List error logs.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public static void listErrorLogs(final Files source)
|
||||
{
|
||||
try
|
||||
{
|
||||
HttpErrorLogs logs = new HttpErrorLogs(source);
|
||||
for (HttpErrorLog log : logs)
|
||||
{
|
||||
System.out.println(log.toStringLog());
|
||||
}
|
||||
}
|
||||
catch (IllegalArgumentException exception)
|
||||
{
|
||||
System.out.println("Bad format line detected. Aborting…");
|
||||
exception.printStackTrace();
|
||||
}
|
||||
catch (DateTimeParseException exception)
|
||||
{
|
||||
System.out.println("Bad datetime format detected. Aborting…");
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
System.out.println("File error detected. Aborting…");
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List ips.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -111,6 +111,33 @@ public class PathCounters extends HashMap<String, PathCounter>
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the count.
|
||||
*
|
||||
* @param path
|
||||
* the path
|
||||
* @param timeMark
|
||||
* the time mark
|
||||
* @return the count
|
||||
*/
|
||||
public long getCount(final String path, final String timeMark)
|
||||
{
|
||||
long result;
|
||||
|
||||
PathCounter counter = get(path, timeMark);
|
||||
if (counter == null)
|
||||
{
|
||||
result = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = counter.getCounter();
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the days values line.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
|
||||
* Copyright (C) 2020-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
|
|
@ -43,9 +43,10 @@ import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
|
|||
import fr.devinsy.statoolinfos.metrics.etherpad.EtherpadProber;
|
||||
import fr.devinsy.statoolinfos.metrics.gitea.GiteaProber;
|
||||
import fr.devinsy.statoolinfos.metrics.gsl.GSLProber;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
|
||||
import fr.devinsy.statoolinfos.metrics.httpaccess.HttpAccessLogAnalyzer;
|
||||
import fr.devinsy.statoolinfos.metrics.httpaccess.HttpAccessLogs;
|
||||
import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogAnalyzer;
|
||||
import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogs;
|
||||
import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber;
|
||||
import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber;
|
||||
import fr.devinsy.statoolinfos.metrics.mumble.MumbleProber;
|
||||
|
|
@ -371,15 +372,32 @@ public class Prober
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe http error log.
|
||||
*
|
||||
* @param configuration
|
||||
* the configuration
|
||||
* @return the path counters
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public static PathCounters probeHttpErrorLog(final Configuration configuration) throws IOException, StatoolInfosException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
logger.info("== Probing HttpErrorLog.");
|
||||
String source = configuration.getProbeHttpErrorLogSource();
|
||||
String pattern = configuration.getProbeHttpErrorLogPattern();
|
||||
String dateTimePattern = configuration.getProbeHttpErrorLogDateTimePattern();
|
||||
logger.info("source=[{}]", source);
|
||||
logger.info("pattern=[{}]", pattern);
|
||||
logger.info("dateTimePattern=[{}]", dateTimePattern);
|
||||
|
||||
result = HttpErrorLogAnalyzer.probe(source);
|
||||
HttpErrorLogs httpErrorLogs = new HttpErrorLogs(FilesUtils.searchByWildcard(source), pattern, dateTimePattern);
|
||||
|
||||
result = HttpErrorLogAnalyzer.probe(httpErrorLogs);
|
||||
|
||||
//
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2021-2023 Christian Pierre MOMON <christian@momon.org>
|
||||
* Copyright (C) 2021-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
|
|
@ -43,6 +43,7 @@ public class HttpErrorLog
|
|||
|
||||
private LocalDateTime time;
|
||||
private String level;
|
||||
private String message;
|
||||
|
||||
/**
|
||||
* Instantiates a new http error log.
|
||||
|
|
@ -68,6 +69,11 @@ public class HttpErrorLog
|
|||
return this.level;
|
||||
}
|
||||
|
||||
public String getMessage()
|
||||
{
|
||||
return this.message;
|
||||
}
|
||||
|
||||
public LocalDateTime getTime()
|
||||
{
|
||||
return this.time;
|
||||
|
|
@ -141,6 +147,11 @@ public class HttpErrorLog
|
|||
this.level = level;
|
||||
}
|
||||
|
||||
public void setMessage(final String message)
|
||||
{
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public void setTime(final LocalDateTime time)
|
||||
{
|
||||
this.time = time;
|
||||
|
|
@ -166,4 +177,29 @@ public class HttpErrorLog
|
|||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* To string log.
|
||||
*
|
||||
* @return the string
|
||||
*/
|
||||
public String toStringLog()
|
||||
{
|
||||
String result;
|
||||
|
||||
// "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+)
|
||||
// \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+)
|
||||
// (?<bodyBytesSent>\\d+) \"(?<referer>.*)\"
|
||||
// \"(?<userAgent>[^\"]*)\".*$");
|
||||
|
||||
// result = String.format("%s %s %s \\[%s\\] \"%s\" %d %d \"%s\"
|
||||
// \"%s\"",
|
||||
result = String.format("%s [%s] %s",
|
||||
this.time.format(DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss.SSS", Locale.ENGLISH)),
|
||||
this.level,
|
||||
this.message);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
|
||||
* Copyright (C) 2020-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
|
|
@ -18,21 +18,15 @@
|
|||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.httperrorlog;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||
import fr.devinsy.statoolinfos.util.LineIterator;
|
||||
import fr.devinsy.strings.StringList;
|
||||
|
||||
/**
|
||||
* The Class HttpErrorLogAnalyzer.
|
||||
|
|
@ -41,11 +35,7 @@ public class HttpErrorLogAnalyzer
|
|||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(HttpErrorLogAnalyzer.class);
|
||||
|
||||
public static final Pattern NGINX_ERROR_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
|
||||
public static final Pattern APACHE_ERROR_PATTERN = Pattern.compile("^\\[(?<time>[^\\]]+)\\]\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
|
||||
|
||||
private PathCounters counters;
|
||||
private int errorCount;
|
||||
|
||||
/**
|
||||
* Instantiates a new http error log analyzer.
|
||||
|
|
@ -53,7 +43,6 @@ public class HttpErrorLogAnalyzer
|
|||
public HttpErrorLogAnalyzer()
|
||||
{
|
||||
this.counters = new PathCounters();
|
||||
this.errorCount += 1;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -72,60 +61,6 @@ public class HttpErrorLogAnalyzer
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param file
|
||||
* the file
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public void probe(final File file) throws IOException
|
||||
{
|
||||
if ((file != null) && (!file.isFile()) || (file.exists()))
|
||||
{
|
||||
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
|
||||
|
||||
//
|
||||
Pattern pattern;
|
||||
if (LineIterator.readFirstLine(file).startsWith("["))
|
||||
{
|
||||
pattern = APACHE_ERROR_PATTERN;
|
||||
}
|
||||
else
|
||||
{
|
||||
pattern = NGINX_ERROR_PATTERN;
|
||||
}
|
||||
|
||||
//
|
||||
LineIterator iterator = new LineIterator(file);
|
||||
while (iterator.hasNext())
|
||||
{
|
||||
String line = iterator.next();
|
||||
|
||||
try
|
||||
{
|
||||
HttpErrorLog log = parseLog(line, pattern);
|
||||
// logger.info("==================");
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
else
|
||||
{
|
||||
probeLog(log);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
|
||||
// exception.printStackTrace();
|
||||
this.errorCount += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe log.
|
||||
*
|
||||
|
|
@ -133,6 +68,8 @@ public class HttpErrorLogAnalyzer
|
|||
* the log
|
||||
*/
|
||||
public void probeLog(final HttpErrorLog log)
|
||||
{
|
||||
if (log != null)
|
||||
{
|
||||
// General HTTP access logs.
|
||||
String year = log.getYear();
|
||||
|
|
@ -144,101 +81,41 @@ public class HttpErrorLogAnalyzer
|
|||
this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
|
||||
|
||||
// TODO metrics.http.errors.php
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @param pattern
|
||||
* the pattern
|
||||
* @return the http error log
|
||||
*/
|
||||
public static HttpErrorLog parseLog(final String line, final Pattern pattern)
|
||||
if (StringUtils.containsIgnoreCase(log.getLevel(), "php"))
|
||||
{
|
||||
HttpErrorLog result;
|
||||
|
||||
if (pattern == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
Matcher matcher = pattern.matcher(line);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = new HttpErrorLog();
|
||||
if (pattern == APACHE_ERROR_PATTERN)
|
||||
{
|
||||
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss.SSSSSS yyyy").withLocale(Locale.ENGLISH)));
|
||||
}
|
||||
else if (pattern == NGINX_ERROR_PATTERN)
|
||||
{
|
||||
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)));
|
||||
}
|
||||
result.setLevel(matcher.group("level"));
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
this.counters.inc("metrics.http.errors.php", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the nginx log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the http error log
|
||||
*/
|
||||
public static HttpErrorLog parseNginxLog(final String line)
|
||||
{
|
||||
HttpErrorLog result;
|
||||
|
||||
// log_format combined '$remote_addr - $remote_user [$time_local] '
|
||||
// '"$request" $status $body_bytes_sent '
|
||||
// '"$http_referer" "$http_user_agent"';
|
||||
// String pattern = "^(?<time>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})
|
||||
// \\[(?<level>[^\\]])\\] .*$";
|
||||
String patternString = "^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$";
|
||||
|
||||
Pattern pattern = Pattern.compile(patternString);
|
||||
|
||||
result = parseLog(line, pattern);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @param logs
|
||||
* the logs
|
||||
* @return the path counters
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public static PathCounters probe(final String source) throws IOException, StatoolInfosException
|
||||
public static PathCounters probe(final HttpErrorLogs logs) throws IOException, StatoolInfosException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
HttpErrorLogAnalyzer analyzer = new HttpErrorLogAnalyzer();
|
||||
|
||||
for (File file : FilesUtils.searchByWildcard(source))
|
||||
HttpErrorLogIterator logIterator = (HttpErrorLogIterator) logs.iterator();
|
||||
while (logIterator.hasNext())
|
||||
{
|
||||
analyzer.probe(file);
|
||||
analyzer.probeLog(logIterator.next());
|
||||
}
|
||||
|
||||
result = analyzer.getCounters();
|
||||
|
||||
StringList timemarks = result.getNowTimeMarks();
|
||||
analyzer.getCounters().set(logIterator.getLogCount(), "metrics.http.logs.count", timemarks);
|
||||
analyzer.getCounters().set(logIterator.getFailedLogCount(), "metrics.http.logs.failed", timemarks);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,275 @@
|
|||
/*
|
||||
* Copyright (C) 2022-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.httperrorlog;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Iterator;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.util.Files;
|
||||
import fr.devinsy.statoolinfos.util.FilesLineIterator;
|
||||
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||
|
||||
/**
|
||||
* The Class HttpErrorLogIterator.
|
||||
*/
|
||||
public class HttpErrorLogIterator implements Iterator<HttpErrorLog>
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(HttpErrorLogIterator.class);
|
||||
|
||||
private FilesLineIterator lineIterator;
|
||||
private Pattern pattern;
|
||||
private DateTimeFormatter dateTimeFormatter;
|
||||
private HttpErrorLog nextLog;
|
||||
private int logCount;
|
||||
private int failedLogCount;
|
||||
|
||||
/**
|
||||
* Instantiates a new http error log iterator.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogIterator(final Files source) throws IOException
|
||||
{
|
||||
this(source, null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new http error log iterator.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @param regex
|
||||
* the regex
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogIterator(final Files source, final String regex) throws IOException
|
||||
{
|
||||
this(source, regex, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new http error log iterator.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @param linePattern
|
||||
* the line pattern
|
||||
* @param dateTimePattern
|
||||
* the date time pattern
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogIterator(final Files source, final String linePattern, final String dateTimePattern) throws IOException
|
||||
{
|
||||
this.lineIterator = new FilesLineIterator(source);
|
||||
this.nextLog = null;
|
||||
this.logCount = 0;
|
||||
this.failedLogCount = 0;
|
||||
|
||||
Pattern defaultLinePattern;
|
||||
DateTimeFormatter defaultDateTimeFormatter;
|
||||
if ((linePattern == null) || (dateTimePattern == null))
|
||||
{
|
||||
if (isApacheHttpErrorLogFiles(source))
|
||||
{
|
||||
defaultLinePattern = HttpErrorLogParser.APACHE_ERROR_PATTERN;
|
||||
defaultDateTimeFormatter = HttpErrorLogParser.APACHE_ERROR_DATETIME_FORMATTER;
|
||||
}
|
||||
else
|
||||
{
|
||||
defaultLinePattern = HttpErrorLogParser.NGINX_ERROR_PATTERN;
|
||||
defaultDateTimeFormatter = HttpErrorLogParser.NGINX_ERROR_DATETIME_FORMATTER;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// These values will never be used.
|
||||
defaultLinePattern = null;
|
||||
defaultDateTimeFormatter = null;
|
||||
}
|
||||
|
||||
if (StringUtils.isBlank(linePattern))
|
||||
{
|
||||
this.pattern = defaultLinePattern;
|
||||
}
|
||||
else
|
||||
{
|
||||
this.pattern = Pattern.compile(linePattern);
|
||||
}
|
||||
|
||||
if (StringUtils.isBlank(dateTimePattern))
|
||||
{
|
||||
this.dateTimeFormatter = defaultDateTimeFormatter;
|
||||
}
|
||||
else
|
||||
{
|
||||
String[] split = dateTimePattern.split("\\|");
|
||||
if (split.length != 2)
|
||||
{
|
||||
throw new IllegalArgumentException("Bad dateTimePattern format: [" + dateTimePattern + "].");
|
||||
}
|
||||
else
|
||||
{
|
||||
this.dateTimeFormatter = DateTimeFormatter.ofPattern(split[0]).withLocale(Locale.forLanguageTag(split[1]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the failed log count.
|
||||
*
|
||||
* @return the failed log count
|
||||
*/
|
||||
public int getFailedLogCount()
|
||||
{
|
||||
return this.failedLogCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the log count.
|
||||
*
|
||||
* @return the log count
|
||||
*/
|
||||
public int getLogCount()
|
||||
{
|
||||
return this.logCount;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.util.Iterator#hasNext()
|
||||
*/
|
||||
@Override
|
||||
public boolean hasNext()
|
||||
{
|
||||
boolean result;
|
||||
|
||||
preload();
|
||||
|
||||
if (this.nextLog == null)
|
||||
{
|
||||
result = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = true;
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.util.Iterator#next()
|
||||
*/
|
||||
@Override
|
||||
public HttpErrorLog next()
|
||||
{
|
||||
HttpErrorLog result;
|
||||
|
||||
preload();
|
||||
|
||||
result = this.nextLog;
|
||||
this.nextLog = null;
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Forward.
|
||||
*/
|
||||
private void preload()
|
||||
{
|
||||
if (this.nextLog == null)
|
||||
{
|
||||
boolean ended = false;
|
||||
while (!ended)
|
||||
{
|
||||
if (this.lineIterator.hasNext())
|
||||
{
|
||||
String line = this.lineIterator.next();
|
||||
this.logCount += 1;
|
||||
|
||||
try
|
||||
{
|
||||
HttpErrorLog log = HttpErrorLogParser.parseLog(line, this.pattern, this.dateTimeFormatter);
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
this.failedLogCount += 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
this.nextLog = log;
|
||||
ended = true;
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
|
||||
this.failedLogCount += 1;
|
||||
// exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
this.nextLog = null;
|
||||
ended = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if is apache http error log files.
|
||||
*
|
||||
* @param files
|
||||
* the files
|
||||
* @return true, if is apache http error log files
|
||||
* @throws IOException
|
||||
*/
|
||||
public static boolean isApacheHttpErrorLogFiles(final Files files) throws IOException
|
||||
{
|
||||
boolean result;
|
||||
|
||||
String firstLine = FilesUtils.readFirstLineNotBlank(files);
|
||||
if (StringUtils.startsWith(firstLine, "["))
|
||||
{
|
||||
result = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = false;
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.httperrorlog;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* The Class HttpErrorLogParser.
|
||||
*/
|
||||
public class HttpErrorLogParser
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(HttpErrorLogParser.class);
|
||||
|
||||
public static final Pattern APACHE_ERROR_PATTERN = Pattern.compile("^\\[(?<time>[^\\]]+)\\]\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
|
||||
public static final Pattern NGINX_ERROR_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
|
||||
|
||||
public static final DateTimeFormatter APACHE_ERROR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss.SSSSSS yyyy").withLocale(Locale.ENGLISH);
|
||||
public static final DateTimeFormatter NGINX_ERROR_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH);
|
||||
|
||||
/**
|
||||
* Instantiates a new http access log parser.
|
||||
*/
|
||||
private HttpErrorLogParser()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the date time.
|
||||
*
|
||||
* @param dateTime
|
||||
* the date time
|
||||
* @param formatter
|
||||
* the formatter
|
||||
* @return the zoned date time
|
||||
*/
|
||||
public static ZonedDateTime parseDateTime(final CharSequence dateTime, final DateTimeFormatter formatter)
|
||||
{
|
||||
ZonedDateTime result;
|
||||
|
||||
if ((dateTime == null) || (formatter == null))
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = ZonedDateTime.parse(dateTime, formatter);
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the http log
|
||||
*/
|
||||
public static HttpErrorLog parseLog(final String line, final Pattern pattern, final DateTimeFormatter dateTimeFormatter)
|
||||
{
|
||||
HttpErrorLog result;
|
||||
|
||||
// log_format combined '$remote_addr - $remote_user [$time_local] '
|
||||
// '"$request" $status $body_bytes_sent '
|
||||
// '"$http_referer" "$http_user_agent"';
|
||||
// String pattern = "^(?<time>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})
|
||||
// \\[(?<level>[^\\]])\\] .*$";
|
||||
// String patternString =
|
||||
// "^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$";
|
||||
|
||||
if ((line == null) || (pattern == null) || (dateTimeFormatter == null))
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
Matcher matcher = pattern.matcher(line);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = new HttpErrorLog();
|
||||
|
||||
result.setTime(LocalDateTime.parse(matcher.group("time"), dateTimeFormatter));
|
||||
result.setLevel(matcher.group("level"));
|
||||
result.setMessage(matcher.group("message"));
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
/*
|
||||
* Copyright (C) 2022-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.httperrorlog;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.util.Files;
|
||||
|
||||
/**
|
||||
* The Class HttpErrorLogs.
|
||||
*/
|
||||
public class HttpErrorLogs implements Iterable<HttpErrorLog>
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(HttpErrorLogs.class);
|
||||
|
||||
private Files source;
|
||||
private String pattern;
|
||||
private String datePattern;
|
||||
|
||||
/**
|
||||
* Instantiates a new http error logs.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogs(final File source) throws IOException
|
||||
{
|
||||
this(new Files(source), null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new http error logs.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @param pattern
|
||||
* the pattern
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogs(final File source, final String pattern) throws IOException
|
||||
{
|
||||
this(new Files(source), pattern, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new http error logs.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @param pattern
|
||||
* the pattern
|
||||
* @param datePattern
|
||||
* the date pattern
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogs(final File source, final String pattern, final String datePattern) throws IOException
|
||||
{
|
||||
this(new Files(source), pattern, datePattern);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new http error logs.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogs(final Files source) throws IOException
|
||||
{
|
||||
this(source, null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new http error logs.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @param pattern
|
||||
* the pattern
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogs(final Files source, final String pattern) throws IOException
|
||||
{
|
||||
this(source, pattern, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instantiates a new http error logs.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @param pattern
|
||||
* the pattern
|
||||
* @param datePattern
|
||||
* the date pattern
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public HttpErrorLogs(final Files source, final String pattern, final String datePattern) throws IOException
|
||||
{
|
||||
this.source = source;
|
||||
this.pattern = pattern;
|
||||
this.datePattern = datePattern;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.lang.Iterable#iterator()
|
||||
*/
|
||||
@Override
|
||||
public Iterator<HttpErrorLog> iterator()
|
||||
{
|
||||
HttpErrorLogIterator result;
|
||||
|
||||
try
|
||||
{
|
||||
result = new HttpErrorLogIterator(this.source, this.pattern, this.datePattern);
|
||||
}
|
||||
catch (IOException exception)
|
||||
{
|
||||
throw new IllegalArgumentException("Error with iterator.", exception);
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,204 @@
|
|||
/*
|
||||
* Copyright (C) 2021-2024 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple key value database.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.httperror;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.core.config.Configurator;
|
||||
import org.apache.logging.log4j.core.config.DefaultConfiguration;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.Configuration;
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.metrics.Prober;
|
||||
import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLog;
|
||||
import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogAnalyzer;
|
||||
import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogIterator;
|
||||
import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogs;
|
||||
import fr.devinsy.statoolinfos.util.Files;
|
||||
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||
|
||||
/**
|
||||
* The Class HttpErrorLogsTest.
|
||||
*/
|
||||
public class HttpErrorLogsTest
|
||||
{
|
||||
private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpErrorLogsTest.class);
|
||||
|
||||
/**
|
||||
* Test 01.
|
||||
*
|
||||
* @throws Exception
|
||||
* the exception
|
||||
*/
|
||||
@Test
|
||||
public void test01() throws Exception
|
||||
{
|
||||
System.out.println(System.getProperty("user.dir"));
|
||||
|
||||
String source = "./test/fr/devinsy/statoolinfos/metrics/httperror/data/paste.libre-service.eu/paste*";
|
||||
Files files = FilesUtils.searchByWildcard(source);
|
||||
for (File file : files)
|
||||
{
|
||||
System.out.println(file);
|
||||
}
|
||||
HttpErrorLogs logs = new HttpErrorLogs(files);
|
||||
HttpErrorLogIterator iterator = (HttpErrorLogIterator) logs.iterator();
|
||||
while (iterator.hasNext())
|
||||
{
|
||||
HttpErrorLog log = iterator.next();
|
||||
System.out.println(log.toStringLog());
|
||||
}
|
||||
|
||||
// System.out.println(iterator.getLogCount());
|
||||
// System.out.println(iterator.getFailedLogCount());
|
||||
|
||||
Assert.assertEquals(315, iterator.getLogCount());
|
||||
Assert.assertEquals(0, iterator.getFailedLogCount());
|
||||
}
|
||||
|
||||
/**
|
||||
* Test 02.
|
||||
*
|
||||
* @throws Exception
|
||||
* the exception
|
||||
*/
|
||||
@Test
|
||||
public void test02() throws Exception
|
||||
{
|
||||
System.out.println(System.getProperty("user.dir"));
|
||||
|
||||
String source = "./test/fr/devinsy/statoolinfos/metrics/httperror/data/paste.libre-service.eu/paste*";
|
||||
Files files = FilesUtils.searchByWildcard(source);
|
||||
for (File file : files)
|
||||
{
|
||||
System.out.println(file);
|
||||
}
|
||||
HttpErrorLogs logs = new HttpErrorLogs(files);
|
||||
|
||||
PathCounters counters = HttpErrorLogAnalyzer.probe(logs);
|
||||
|
||||
for (String prefix : counters.getPrefixes())
|
||||
{
|
||||
System.out.println(prefix);
|
||||
|
||||
}
|
||||
|
||||
System.out.println("Prefix count: " + counters.getPrefixes().size());
|
||||
System.out.println("metrics.http.errors.2023.months=" + counters.get("metrics.http.errors", "2023-08").getCounter());
|
||||
|
||||
Assert.assertEquals(2, counters.getPrefixes().size());
|
||||
Assert.assertEquals(25, counters.getCount("metrics.http.errors", "2023-08"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test 03.
|
||||
*
|
||||
* @throws Exception
|
||||
* the exception
|
||||
*/
|
||||
@Test
|
||||
public void test03() throws Exception
|
||||
{
|
||||
System.out.println(System.getProperty("user.dir"));
|
||||
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.add("conf.probe.types", "HttpErrorLog");
|
||||
configuration.add("conf.probe.httperrorlog.file", "./test/fr/devinsy/statoolinfos/metrics/httperror/data/paste.libre-service.eu/paste*");
|
||||
// configuration.add("conf.probe.httperrorlog.pattern", "");
|
||||
// configuration.add("conf.probe.httperrorlog.dateTimePattern", "");
|
||||
|
||||
PathCounters counters = Prober.probeHttpErrorLog(configuration);
|
||||
|
||||
// for (String counter : counters.keySet())
|
||||
// {
|
||||
// System.out.println(counter + " " +
|
||||
// counters.get(counter).getCounter());
|
||||
// }
|
||||
|
||||
Assert.assertEquals(660, counters.size());
|
||||
|
||||
//
|
||||
Assert.assertEquals(210, counters.getCount("metrics.http.errors", "2023"));
|
||||
Assert.assertEquals(16, counters.getCount("metrics.http.errors", "2023-10"));
|
||||
Assert.assertEquals(7, counters.getCount("metrics.http.errors", "2023-W31"));
|
||||
Assert.assertEquals(1, counters.getCount("metrics.http.errors", "2023-05-20"));
|
||||
Assert.assertEquals(",21,19,23,26,23,22,25,10,16,9,16", counters.getMonthsValuesLine("metrics.http.errors", "2023"));
|
||||
Assert.assertEquals(",,,,3,4,4,9,4,5,4,4,5,5,3,6,7,8,4,7,4,5,5,6,5,5,6,6,4,6,7,6,6,5,1,4,3,2,2,2,6,4,2,3,3,2,,4,3,5,2,4", counters.getWeeksValuesLine("metrics.http.errors", "2023"));
|
||||
|
||||
Assert.assertEquals(190, counters.getCount("metrics.http.errors.php", "2023"));
|
||||
Assert.assertEquals(7, counters.getCount("metrics.http.errors", "2023-W31"));
|
||||
Assert.assertEquals(22, counters.getCount("metrics.http.errors.php", "2023-07"));
|
||||
Assert.assertEquals(",17,19,22,20,22,22,23,10,15,8,12", counters.getMonthsValuesLine("metrics.http.errors.php", "2023"));
|
||||
Assert.assertEquals(",,,,3,3,4,6,4,5,4,4,5,5,3,6,6,4,3,7,3,5,5,6,4,5,6,6,4,6,6,5,6,5,1,4,3,2,2,2,5,4,2,3,3,1,,3,3,2,2,4", counters.getWeeksValuesLine("metrics.http.errors.php", "2023"));
|
||||
|
||||
//
|
||||
Assert.assertEquals(105, counters.getCount("metrics.http.errors", "2024"));
|
||||
Assert.assertEquals(19, counters.getCount("metrics.http.errors", "2024-02"));
|
||||
Assert.assertEquals(7, counters.getCount("metrics.http.errors", "2024-W17"));
|
||||
Assert.assertEquals(2, counters.getCount("metrics.http.errors", "2024-03-15"));
|
||||
|
||||
Assert.assertEquals("9,19,18,23,13,17,6", counters.getMonthsValuesLine("metrics.http.errors", "2024"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test04() throws Exception
|
||||
{
|
||||
System.out.println(System.getProperty("user.dir"));
|
||||
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.add("conf.probe.types", "HttpErrorLog");
|
||||
configuration.add("conf.probe.httperrorlog.file", "/home/cpm/Projets/StatoolInfos/EnvTest/logs/pastechaprilorg-error.log.gz");
|
||||
// configuration.add("conf.probe.httperrorlog.pattern", "TODO");
|
||||
// configuration.add("conf.probe.httperrorlog.dateTimePattern", "TODO");
|
||||
|
||||
PathCounters counters = Prober.probeHttpErrorLog(configuration);
|
||||
|
||||
Assert.assertEquals(84, counters.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* After class.
|
||||
*
|
||||
* @throws StatoolInfosException
|
||||
* the Juga exception
|
||||
*/
|
||||
@AfterClass
|
||||
public static void afterClass() throws StatoolInfosException
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Before class.
|
||||
*
|
||||
* @throws StatoolInfosException
|
||||
* the Juga exception
|
||||
*/
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws StatoolInfosException
|
||||
{
|
||||
Configurator.initialize(new DefaultConfiguration());
|
||||
Configurator.setRootLevel(Level.DEBUG);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue