statool/statoolinfosweb
statool/statoolinfosweb
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Activity

Added HttpAccessLogDateTimePattern management (#sleto).

Browse source
This commit is contained in:
Christian P. MOMON 2023-07-03 17:40:10 +02:00
parent 8619c59a65
commit 02281b1ab6
8 changed files with 120 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -281,6 +281,10 @@ conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.
# Default: ^(?<remoteAddress>[a-fA-F0-9\:\.]+) - (?<remoteUser>[^\[]+) \[(?<time>[^\]]+)\] "(?<request>.*)" (?<status>\d+) (?<bodyBytesSent>\d+) "(?<referer>.*)" "(?<userAgent>[^"]*)".*$
conf.probe.httpaccesslog.pattern=
# Custom datetime access log pattern (https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html#patterns).
# Default: dd/MMM/yyyy:HH:mm:ss Z|EN
conf.prob.httpaccesslog.datetimepattern=
# Filter request with Java regex (without the initial '^' and without the final '$', see https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html).
# Example: pathfilter=/.*well-known.*
conf.probe.httpaccesslog.pathfilter=

15
src/fr/devinsy/statoolinfos/core/Configuration.java
View file

@ -354,6 +354,21 @@ public class Configuration extends PathPropertyList
return result;
}
/**
* Gets the probe http access log date pattern.
*
* @return the probe http access log date pattern
*/
public String getProbeHttpAccessLogDateTimePattern()
{
String result;
result = get("conf.probe.httpaccesslog.datetimepattern");
//
return result;
}
/**
* Gets the probe http access log path filter.
*

4
src/fr/devinsy/statoolinfos/core/StatoolInfos.java
View file

@ -295,13 +295,15 @@ public class StatoolInfos
logger.info("== Testing HttpAccessLog lines.");
String source = configuration.getProbeHttpAccessLogSource();
String dateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
String pattern = configuration.getProbeHttpAccessLogPattern();
String pathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("source=[{}]", source);
logger.info("pattern=[{}]", pattern);
logger.info("dateTimePattern=[{}]", dateTimePattern);
logger.info("pathFilter=[{}]", pathFilter);
HttpAccessLogs logs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, pathFilter);
HttpAccessLogs logs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, dateTimePattern, pathFilter);
for (HttpAccessLog log : logs)
{

20
src/fr/devinsy/statoolinfos/metrics/Prober.java
View file

@ -227,11 +227,13 @@ public class Prober
String httpLogs = configuration.getProbeHttpAccessLogSource();
String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
String httpAccessLogDateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
String httpAccessPathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("httpLogs=[{}]", httpLogs);
logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
logger.info("httpAccessDateTimePattern=[{}]", httpAccessLogDateTimePattern);
logger.info("httpAccessPathFilter=[{}]", httpAccessPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessLogDateTimePattern, httpAccessPathFilter);
result = EtherpadProber.probe(httpAccessLogs, FilesUtils.searchByWildcard(logs), database);
@ -296,11 +298,13 @@ public class Prober
String httpLogs = configuration.getProbeHttpAccessLogSource();
String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
String httpAccessLogDateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("httpLogs=[{}]", httpLogs);
logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
logger.info("httpAccessDateTimePattern=[{}]", httpAccessLogDateTimePattern);
logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessLogDateTimePattern, httpAccessLogPathFilter);
result = GiteaProber.probe(httpAccessLogs, apiURL, apiToken, dataDirectory, database);
@ -352,12 +356,14 @@ public class Prober
logger.info("== Probing HttpAccessLog.");
String source = configuration.getProbeHttpAccessLogSource();
String pattern = configuration.getProbeHttpAccessLogPattern();
String dateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
String pathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("source=[{}]", source);
logger.info("pattern=[{}]", pattern);
logger.info("dateTimePattern=[{}]", dateTimePattern);
logger.info("path=[{}]", pathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, pathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, dateTimePattern, pathFilter);
result = HttpAccessLogAnalyzer.probe(httpAccessLogs);
@ -398,11 +404,13 @@ public class Prober
String httpAccessSource = configuration.getProbeHttpAccessLogSource();
String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
String httpAccessLogDateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("httpAccessSource=[{}]", httpAccessSource);
logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
logger.info("httpAccessDateTimePattern=[{}]", httpAccessLogDateTimePattern);
logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessSource), httpAccessLogPattern, httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessSource), httpAccessLogPattern, httpAccessLogDateTimePattern, httpAccessLogPathFilter);
File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles");
logger.info("dataDirectory=[{}]", dataDirectory);
@ -485,11 +493,13 @@ public class Prober
logger.info("== Probing Privatebin.");
String httpAccessLogSource = configuration.getProbeHttpAccessLogSource();
String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
String httpAccessLogDateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("source=[{}]", httpAccessLogSource);
logger.info("pattern=[{}]", httpAccessLogPattern);
logger.info("httpAccessDateTimePattern=[{}]", httpAccessLogDateTimePattern);
logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessLogSource), httpAccessLogPattern, httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessLogSource), httpAccessLogPattern, httpAccessLogDateTimePattern, httpAccessLogPathFilter);
File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
logger.info("dataDirectory=[{}]", dataDirectory);

40
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java
View file

@ -19,7 +19,9 @@
package fr.devinsy.statoolinfos.metrics.http;
import java.io.IOException;
import java.time.format.DateTimeFormatter;
import java.util.Iterator;
import java.util.Locale;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
@ -38,6 +40,7 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
private FilesLineIterator lineIterator;
private Pattern pattern;
private DateTimeFormatter dateTimeFormatter;
private Pattern requestFilter;
private HttpAccessLog nextLog;
private int errorCount;
@ -53,7 +56,7 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
*/
public HttpAccessLogIterator(final Files source) throws IOException
{
this(source, null, null);
this(source, null, null, null);
}
/**
@ -68,31 +71,54 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
*/
public HttpAccessLogIterator(final Files source, final String regex) throws IOException
{
this(source, regex, null);
this(source, regex, null, null);
}
/**
* Instantiates a new http log iterator.
* Instantiates a new http access log iterator.
*
* @param source
* the source
* @param linePattern
* the line pattern
* @param dateTimePattern
* the date time pattern
* @param pathFilter
* the path filter
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogIterator(final Files source, final String regex, final String pathFilter) throws IOException
public HttpAccessLogIterator(final Files source, final String linePattern, final String dateTimePattern, final String pathFilter) throws IOException
{
this.lineIterator = new FilesLineIterator(source);
this.nextLog = null;
this.errorCount = 0;
this.ignoredLineCount = 0;
if (StringUtils.isBlank(regex))
if (StringUtils.isBlank(linePattern))
{
this.pattern = HttpAccessLogParser.COMBINED_PATTERN;
}
else
{
this.pattern = Pattern.compile(regex);
this.pattern = Pattern.compile(linePattern);
}
if (StringUtils.isBlank(dateTimePattern))
{
this.dateTimeFormatter = HttpAccessLogParser.DATETIME_FORMATTER;
}
else
{
String[] split = dateTimePattern.split("\\|");
if (split.length != 2)
{
throw new IllegalArgumentException("Bad dateTimePattern format: [" + dateTimePattern + "].");
}
else
{
this.dateTimeFormatter = DateTimeFormatter.ofPattern(split[0]).withLocale(Locale.forLanguageTag(split[1]));
}
}
if (StringUtils.isBlank(pathFilter))
@ -183,7 +209,7 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
try
{
HttpAccessLog log = HttpAccessLogParser.parseLog(line, this.pattern);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, this.pattern, this.dateTimeFormatter);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);

42
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogParser.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -21,6 +21,7 @@ package fr.devinsy.statoolinfos.metrics.http;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -41,6 +42,8 @@ public class HttpAccessLogParser
public static final Pattern COMBINED_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-fA-F0-9\\:\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>.*)\".*$");
public static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH);
/**
* Instantiates a new http access log parser.
*/
@ -48,6 +51,39 @@ public class HttpAccessLogParser
{
}
/**
* Parses the date time.
*
* @param dateTime
* the date time
* @param formatter
* the formatter
* @return the zoned date time
*/
public static ZonedDateTime parseDateTime(final CharSequence dateTime, final DateTimeFormatter formatter)
{
ZonedDateTime result;
if ((formatter == null) || (formatter == DATETIME_FORMATTER))
{
result = ZonedDateTime.parse(dateTime, DATETIME_FORMATTER);
}
else
{
try
{
result = ZonedDateTime.parse(dateTime, formatter);
}
catch (DateTimeParseException exception)
{
result = ZonedDateTime.parse(dateTime, DATETIME_FORMATTER);
}
}
//
return result;
}
/**
* Parses the log.
*
@ -55,7 +91,7 @@ public class HttpAccessLogParser
* the line
* @return the http log
*/
public static HttpAccessLog parseLog(final String line, final Pattern pattern)
public static HttpAccessLog parseLog(final String line, final Pattern pattern, final DateTimeFormatter dateTimeFormatter)
{
HttpAccessLog result;
@ -68,7 +104,7 @@ public class HttpAccessLogParser
result = new HttpAccessLog();
result.setIp(matcher.group("remoteAddress"));
result.setRemoteUser(matcher.group("remoteUser"));
result.setTime(ZonedDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
result.setTime(parseDateTime(matcher.group("time"), dateTimeFormatter));
result.setRequest(matcher.group("request"));
result.setStatus(HttpStatusTable.instance().get(matcher.group("status")));
result.setBodyBytesSent(Long.valueOf(matcher.group("bodyBytesSent")));

16
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java
View file

@ -36,6 +36,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
private Files source;
private String pattern;
private String datePattern;
private String pathFilter;
/**
@ -48,7 +49,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
*/
public HttpAccessLogs(final File source) throws IOException
{
this(new Files(source), null, null);
this(new Files(source), null, null, null);
}
/**
@ -63,7 +64,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
*/
public HttpAccessLogs(final File source, final String pattern) throws IOException
{
this(new Files(source), pattern, null);
this(new Files(source), pattern, null, null);
}
/**
@ -80,7 +81,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
*/
public HttpAccessLogs(final File source, final String pattern, final String pathFilter) throws IOException
{
this(new Files(source), pattern, null);
this(new Files(source), pattern, null, null);
}
/**
@ -93,7 +94,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
*/
public HttpAccessLogs(final Files source) throws IOException
{
this(source, null, null);
this(source, null, null, null);
}
/**
@ -106,7 +107,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
*/
public HttpAccessLogs(final Files source, final String pattern) throws IOException
{
this(source, pattern, null);
this(source, pattern, null, null);
}
/**
@ -121,11 +122,12 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogs(final Files source, final String pattern, final String pathFilter) throws IOException
public HttpAccessLogs(final Files source, final String pattern, final String datePattern, final String pathFilter) throws IOException
{
this.source = source;
this.pattern = pattern;
this.pathFilter = pathFilter;
this.datePattern = datePattern;
}
/* (non-Javadoc)
@ -138,7 +140,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
try
{
result = new HttpAccessLogIterator(this.source, this.pattern, this.pathFilter);
result = new HttpAccessLogIterator(this.source, this.pattern, this.datePattern, this.pathFilter);
}
catch (IOException exception)
{

4
src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2021-2023 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -97,7 +97,7 @@ public class HttpLogIterator
try
{
String line = this.iterator.next();
result = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
result = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN, HttpAccessLogParser.DATETIME_FORMATTER);
}
catch (Exception exception)
{