Added http error log analyze.
This commit is contained in:
parent
45d797ea06
commit
4ee90a3671
8 changed files with 294 additions and 63 deletions
|
|
@ -114,11 +114,7 @@ public class Prober
|
|||
}
|
||||
|
||||
//
|
||||
{
|
||||
PathCounter counter = counters.get("metrics.http.hits", "2020");
|
||||
logger.info("counter=" + counter.getCounter());
|
||||
logger.info("size={}", counters.size());
|
||||
}
|
||||
logger.info("size={}", counters.size());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -31,11 +31,11 @@ import fr.devinsy.strings.StringList;
|
|||
import fr.devinsy.strings.StringsUtils;
|
||||
|
||||
/**
|
||||
* The Class HttpLog.
|
||||
* The Class HttpAccessLog.
|
||||
*/
|
||||
public class HttpLog
|
||||
public class HttpAccessLog
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(HttpLog.class);
|
||||
private static Logger logger = LoggerFactory.getLogger(HttpAccessLog.class);
|
||||
|
||||
public static final Pattern IPV4_PATTERN = Pattern.compile("\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}");
|
||||
public static final Pattern IPV6_PATTERN = Pattern.compile("([0-9a-f]{1,4}:{1,2}){4,7}([0-9a-f]){1,4}", Pattern.CASE_INSENSITIVE);
|
||||
|
|
@ -50,9 +50,9 @@ public class HttpLog
|
|||
private UserAgent userAgent;
|
||||
|
||||
/**
|
||||
* Instantiates a new http log.
|
||||
* Instantiates a new http access log.
|
||||
*/
|
||||
public HttpLog()
|
||||
public HttpAccessLog()
|
||||
{
|
||||
this.remoteAddress = null;
|
||||
this.remoteUser = null;
|
||||
|
|
@ -124,8 +124,7 @@ public class HttpAccessLogAnalyzer
|
|||
*/
|
||||
public void probeLine(final String line)
|
||||
{
|
||||
|
||||
HttpLog log = parseNginxCombinedLog(line);
|
||||
HttpAccessLog log = parseNginxCombinedLog(line);
|
||||
|
||||
// logger.info("==================");
|
||||
if (log == null)
|
||||
|
|
@ -171,7 +170,7 @@ public class HttpAccessLogAnalyzer
|
|||
}
|
||||
|
||||
// metrics.http.pages.* =
|
||||
if (StringUtils.endsWithAny(log.getRequest(), ".html", ".htm", ".xhtml", ".cgi"))
|
||||
if (StringUtils.endsWithAny(log.getRequest(), ".html", ".htm", ".xhtml", ".cgi", "/"))
|
||||
{
|
||||
this.counters.inc("metrics.http.pages", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
|
|
@ -223,38 +222,16 @@ public class HttpAccessLogAnalyzer
|
|||
* the line
|
||||
* @return the http log
|
||||
*/
|
||||
public static HttpLog parseNginxCombinedLog(final String line)
|
||||
public static HttpAccessLog parseLog(final String line, final String pattern)
|
||||
{
|
||||
HttpLog result;
|
||||
|
||||
// log_format combined '$remote_addr - $remote_user [$time_local] '
|
||||
// '"$request" $status $body_bytes_sent '
|
||||
// '"$http_referer" "$http_user_agent"';
|
||||
String combinedPattern = "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$";
|
||||
|
||||
result = parseLog(line, combinedPattern);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the http log
|
||||
*/
|
||||
public static HttpLog parseLog(final String line, final String pattern)
|
||||
{
|
||||
HttpLog result;
|
||||
HttpAccessLog result;
|
||||
|
||||
Pattern combined = Pattern.compile(pattern);
|
||||
|
||||
Matcher matcher = combined.matcher(line);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = new HttpLog();
|
||||
result = new HttpAccessLog();
|
||||
result.setRemoteAddress(matcher.group("remoteAddress"));
|
||||
result.setRemoteUser(matcher.group("remoteUser"));
|
||||
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
|
||||
|
|
@ -273,6 +250,28 @@ public class HttpAccessLogAnalyzer
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the http log
|
||||
*/
|
||||
public static HttpAccessLog parseNginxCombinedLog(final String line)
|
||||
{
|
||||
HttpAccessLog result;
|
||||
|
||||
// log_format combined '$remote_addr - $remote_user [$time_local] '
|
||||
// '"$request" $status $body_bytes_sent '
|
||||
// '"$http_referer" "$http_user_agent"';
|
||||
String combinedPattern = "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$";
|
||||
|
||||
result = parseLog(line, combinedPattern);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
|
|
|
|||
163
src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLog.java
Normal file
163
src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLog.java
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
/*
|
||||
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.http;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.metrics.TimeMarkUtils;
|
||||
import fr.devinsy.strings.StringList;
|
||||
|
||||
/**
|
||||
* The Class HttpErrorLog.
|
||||
*/
|
||||
public class HttpErrorLog
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(HttpErrorLog.class);
|
||||
|
||||
private LocalDateTime time;
|
||||
private String level;
|
||||
|
||||
/**
|
||||
* Instantiates a new http error log.
|
||||
*/
|
||||
public HttpErrorLog()
|
||||
{
|
||||
this.time = null;
|
||||
this.level = null;
|
||||
}
|
||||
|
||||
public String getDate()
|
||||
{
|
||||
String result;
|
||||
|
||||
result = this.time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
public String getLevel()
|
||||
{
|
||||
return this.level;
|
||||
}
|
||||
|
||||
public LocalDateTime getTime()
|
||||
{
|
||||
return this.time;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year.
|
||||
*
|
||||
* @return the year
|
||||
*/
|
||||
public String getYear()
|
||||
{
|
||||
String result;
|
||||
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearOf(this.time);
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year month.
|
||||
*
|
||||
* @return the year month
|
||||
*/
|
||||
public String getYearMonth()
|
||||
{
|
||||
String result;
|
||||
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearMonthOf(this.time);
|
||||
}
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year week.
|
||||
*
|
||||
* @return the year week
|
||||
*/
|
||||
public String getYearWeek()
|
||||
{
|
||||
String result;
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearWeekOf(this.time);
|
||||
}
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
public void setLevel(final String level)
|
||||
{
|
||||
this.level = level;
|
||||
}
|
||||
|
||||
public void setTime(final LocalDateTime time)
|
||||
{
|
||||
this.time = time;
|
||||
}
|
||||
|
||||
/**
|
||||
* To string.
|
||||
*
|
||||
* @return the string
|
||||
*/
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
String result;
|
||||
|
||||
StringList buffer = new StringList();
|
||||
|
||||
buffer.append("[time=").append(this.time).append("]");
|
||||
buffer.append("[level=").append(this.level).append("]");
|
||||
|
||||
result = buffer.toString();
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,8 @@ import java.io.IOException;
|
|||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -107,20 +109,91 @@ public class HttpErrorLogAnalyzer
|
|||
*/
|
||||
public void probeLine(final String line)
|
||||
{
|
||||
HttpErrorLog log = parseNginxLog(line);
|
||||
|
||||
// logger.info("==================");
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
else
|
||||
{
|
||||
// logger.info("LINE IS MATCHING [{}]", log);
|
||||
// logger.info(log.getHttpUserAgent().toString());
|
||||
|
||||
// General HTTP access logs.
|
||||
String year = log.getYear();
|
||||
String yearMonth = log.getYearMonth();
|
||||
String yearWeek = log.getYearWeek();
|
||||
String date = log.getDate();
|
||||
|
||||
// metrics.http.hits
|
||||
this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
|
||||
// TODO metrics.http.errors.php
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @param pattern
|
||||
* the pattern
|
||||
* @return the http error log
|
||||
*/
|
||||
public static HttpErrorLog parseLog(final String line, final Pattern pattern)
|
||||
{
|
||||
HttpErrorLog result;
|
||||
|
||||
if (pattern == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
Matcher matcher = pattern.matcher(line);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = new HttpErrorLog();
|
||||
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)));
|
||||
result.setLevel(matcher.group("level"));
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
LocalDateTime date;
|
||||
return result;
|
||||
}
|
||||
|
||||
date = LocalDateTime.now();
|
||||
/**
|
||||
* Parses the nginx log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the http error log
|
||||
*/
|
||||
public static HttpErrorLog parseNginxLog(final String line)
|
||||
{
|
||||
HttpErrorLog result;
|
||||
|
||||
String year = date.format(DateTimeFormatter.ofPattern("yyyy", Locale.FRANCE));
|
||||
String month = date.format(DateTimeFormatter.ofPattern("yyyy-MM", Locale.FRANCE));
|
||||
String week = date.format(DateTimeFormatter.ofPattern("yyyyWW", Locale.FRANCE));
|
||||
String day = date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
|
||||
// log_format combined '$remote_addr - $remote_user [$time_local] '
|
||||
// '"$request" $status $body_bytes_sent '
|
||||
// '"$http_referer" "$http_user_agent"';
|
||||
// String pattern = "^(?<time>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})
|
||||
// \\[(?<level>[^\\]])\\] .*$";
|
||||
String patternString = "^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$";
|
||||
|
||||
// metrics.http.errors
|
||||
this.counters.inc("metrics.http.errors", year, month, week, day);
|
||||
Pattern pattern = Pattern.compile(patternString);
|
||||
|
||||
// metrics.http.errors.php
|
||||
result = parseLog(line, pattern);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -92,9 +92,9 @@ public class HttpLogIterator
|
|||
* @return the http log
|
||||
* @throws IOException
|
||||
*/
|
||||
public HttpLog next() throws IOException
|
||||
public HttpAccessLog next() throws IOException
|
||||
{
|
||||
HttpLog result;
|
||||
HttpAccessLog result;
|
||||
|
||||
try
|
||||
{
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ public class VisitCounters extends HashMap<String, Visits>
|
|||
* the time mark
|
||||
* @return the string
|
||||
*/
|
||||
public String computeKey(final HttpLog log)
|
||||
public String computeKey(final HttpAccessLog log)
|
||||
{
|
||||
String result;
|
||||
|
||||
|
|
@ -136,7 +136,7 @@ public class VisitCounters extends HashMap<String, Visits>
|
|||
* @param log
|
||||
* the log
|
||||
*/
|
||||
public void putVisit(final HttpLog log)
|
||||
public void putVisit(final HttpAccessLog log)
|
||||
{
|
||||
if (log != null)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ public class VisitCountersTest
|
|||
VisitCounters visitCounters = new VisitCounters();
|
||||
|
||||
//
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 23, 0));
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
|
|
@ -83,14 +83,14 @@ public class VisitCountersTest
|
|||
|
||||
//
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0));
|
||||
visitCounters.putVisit(log);
|
||||
}
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20));
|
||||
|
|
@ -123,21 +123,21 @@ public class VisitCountersTest
|
|||
|
||||
//
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0));
|
||||
visitCounters.putVisit(log);
|
||||
}
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20));
|
||||
visitCounters.putVisit(log);
|
||||
}
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 55));
|
||||
|
|
@ -170,21 +170,21 @@ public class VisitCountersTest
|
|||
|
||||
//
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0));
|
||||
visitCounters.putVisit(log);
|
||||
}
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 55));
|
||||
visitCounters.putVisit(log);
|
||||
}
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20));
|
||||
|
|
@ -217,21 +217,21 @@ public class VisitCountersTest
|
|||
|
||||
//
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0));
|
||||
visitCounters.putVisit(log);
|
||||
}
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 35));
|
||||
visitCounters.putVisit(log);
|
||||
}
|
||||
{
|
||||
HttpLog log = new HttpLog();
|
||||
HttpAccessLog log = new HttpAccessLog();
|
||||
log.setRemoteAddress("192.168.1.1");
|
||||
log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
|
||||
log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20));
|
||||
|
|
|
|||
Loading…
Reference in a new issue