statool/statoolinfos
statool/statoolinfos
1
0
Fork 0
Code Issues Pull requests Releases 8 Activity

Improved help message, custom pattern log support and Apache patterns.

Browse source
This commit is contained in:
Christian P. MOMON 2021-12-08 00:39:48 +01:00
parent 11ba05212c
commit 8a563ff194
10 changed files with 380 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java
View file

@ -25,7 +25,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.LogFilter;
import fr.devinsy.statoolinfos.core.BotFilter;
import fr.devinsy.statoolinfos.core.StatoolInfos;
import fr.devinsy.statoolinfos.util.BuildInformation;
import fr.devinsy.statoolinfos.util.Chrono;
@ -104,20 +104,22 @@ public final class StatoolInfosCLI
message.appendln("Usage:");
message.appendln(" statoolinfos [ -h | -help | --help ]");
message.appendln(" statoolinfos [ -v | -version | --version ]");
message.appendln(" statoolinfos build <fileordirectory> build property files from conf and input");
message.appendln(" statoolinfos clear <fileordirectory> remove property files from conf");
message.appendln(" statoolinfos crawl <fileordirectory> crawl all file from conf and input");
message.appendln(" statoolinfos format <fileordirectory> format in tiny way");
message.appendln(" statoolinfos htmlize <fileordirectory> generate web pages from conf");
message.appendln();
message.appendln(" statoolinfos build <configurationfile> build property files from conf and input");
message.appendln(" statoolinfos clear <configurationfile> remove property files from conf");
message.appendln(" statoolinfos crawl <configurationfile> crawl all file from conf and input");
message.appendln(" statoolinfos htmlize <configurationfile> generate web pages from conf");
message.appendln(" statoolinfos probe <configurationfile> generate metrics files from conf");
message.appendln(" statoolinfos uptime <configurationfile> update uptime journal");
message.appendln();
message.appendln(" statoolinfos format <fileordirectory> format property files in tiny way");
message.appendln(" statoolinfos list ip [-bot|-nobot] <fileordirectory> generate ip list from log file");
message.appendln(" statoolinfos list ua [-bot|-nobot] <fileordirectory> generate user agent list from log file");
message.appendln(" statoolinfos list visitors [-bot|-nobot] <fileordirectory> generate visitors (ip+ua) list from log file");
message.appendln(" statoolinfos probe <fileordirectory> generate metrics files from conf");
message.appendln(" statoolinfos stat ip [-bot|-nobot] <fileordirectory> generate stats about ip from log file");
message.appendln(" statoolinfos stat ua [-bot|-nobot] <fileordirectory> generate stats about user agent from log file");
message.appendln(" statoolinfos stat visitors [-bot|-nobot] <fileordirectory> generate stats about visitors (ip+ua) from log file");
message.appendln(" statoolinfos tagdate <fileordirectory> update the file.datetime file");
message.appendln(" statoolinfos uptime <fileordirectory> update uptime journal");
System.out.println(message.toString());
}
@ -200,21 +202,21 @@ public final class StatoolInfosCLI
* the source
* @return the log filter
*/
private static LogFilter parseLogFilterOption(final String source)
private static BotFilter parseLogFilterOption(final String source)
{
LogFilter result;
BotFilter result;
if (StringUtils.equals(source, "-all"))
{
result = LogFilter.ALL;
result = BotFilter.ALL;
}
else if (StringUtils.equals(source, "-bot"))
{
result = LogFilter.BOT;
result = BotFilter.BOT;
}
else if (StringUtils.equals(source, "-nobot"))
{
result = LogFilter.NOBOT;
result = BotFilter.NOBOT;
}
else
{
@ -271,51 +273,43 @@ public final class StatoolInfosCLI
{
displayVersion();
}
else if (isMatching(args, "build", "\\s*.+\\s*"))
else if (isMatching(args, "build", "\\s*.+\\.conf\\s*"))
{
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs)
File configurationFile = new File(StringUtils.trim(args[1]));
try
{
try
{
StatoolInfos.build(input);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
}
StatoolInfos.build(configurationFile);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
}
}
else if (isMatching(args, "clear", "\\s*.+\\s*"))
else if (isMatching(args, "clear", "\\s*.+\\.conf\\s*"))
{
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs)
File configurationFile = new File(StringUtils.trim(args[1]));
try
{
try
{
StatoolInfos.clear(input);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
}
StatoolInfos.clear(configurationFile);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
}
}
else if (isMatching(args, "crawl", "\\s*.+\\s*"))
else if (isMatching(args, "crawl", "\\s*.+\\.conf\\s*"))
{
Chrono chrono = new Chrono().start();
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs)
File configurationFile = new File(StringUtils.trim(args[1]));
try
{
try
{
StatoolInfos.crawl(input);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
StatoolInfos.crawl(configurationFile);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
System.out.println(chrono.format());
}
@ -335,21 +329,18 @@ public final class StatoolInfosCLI
}
}
}
else if (isMatching(args, "htmlize", "\\s*.+\\s*"))
else if (isMatching(args, "htmlize", "\\s*.+\\.conf\\s*"))
{
Chrono chrono = new Chrono().start();
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs)
File configurationFile = new File(StringUtils.trim(args[1]));
try
{
try
{
StatoolInfos.htmlize(input);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
StatoolInfos.htmlize(configurationFile);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
System.out.println(chrono.format());
}
@ -357,11 +348,11 @@ public final class StatoolInfosCLI
{
File source = new File(args[2]);
StatoolInfos.listIps(source, LogFilter.ALL);
StatoolInfos.listIps(source, BotFilter.ALL);
}
else if (isMatching(args, "list", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{
LogFilter filter = parseLogFilterOption(args[2]);
BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]);
StatoolInfos.listIps(source, filter);
@ -370,11 +361,11 @@ public final class StatoolInfosCLI
{
File source = new File(args[2]);
StatoolInfos.listUserAgents(source, LogFilter.ALL);
StatoolInfos.listUserAgents(source, BotFilter.ALL);
}
else if (isMatching(args, "list", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{
LogFilter filter = parseLogFilterOption(args[2]);
BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]);
StatoolInfos.listUserAgents(source, filter);
@ -383,40 +374,37 @@ public final class StatoolInfosCLI
{
File source = new File(args[2]);
StatoolInfos.listVisitors(source, LogFilter.ALL);
StatoolInfos.listVisitors(source, BotFilter.ALL);
}
else if (isMatching(args, "list", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{
LogFilter filter = parseLogFilterOption(args[2]);
BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]);
StatoolInfos.listVisitors(source, filter);
}
else if (isMatching(args, "probe", "\\s*.+\\s*"))
else if (isMatching(args, "probe", "\\s*.+\\.conf\\s*"))
{
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs)
File configurationFile = new File(StringUtils.trim(args[1]));
try
{
try
{
StatoolInfos.probe(input);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
StatoolInfos.probe(configurationFile);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
}
else if (isMatching(args, "stat", "ip", "\\s*\\S+\\s*"))
{
File source = new File(args[2]);
StatoolInfos.statIps(source, LogFilter.ALL);
StatoolInfos.statIps(source, BotFilter.ALL);
}
else if (isMatching(args, "stat", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{
LogFilter filter = parseLogFilterOption(args[2]);
BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]);
StatoolInfos.statIps(source, filter);
@ -424,11 +412,11 @@ public final class StatoolInfosCLI
else if (isMatching(args, "stat", "(useragent|ua)", "\\s*\\S+\\s*"))
{
File source = new File(args[2]);
StatoolInfos.statUserAgents(source, LogFilter.ALL);
StatoolInfos.statUserAgents(source, BotFilter.ALL);
}
else if (isMatching(args, "stat", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{
LogFilter filter = parseLogFilterOption(args[2]);
BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]);
StatoolInfos.statUserAgents(source, filter);
@ -437,11 +425,11 @@ public final class StatoolInfosCLI
{
File source = new File(args[2]);
StatoolInfos.statVisitors(source, LogFilter.ALL);
StatoolInfos.statVisitors(source, BotFilter.ALL);
}
else if (isMatching(args, "stat", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{
LogFilter filter = parseLogFilterOption(args[2]);
BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]);
StatoolInfos.statVisitors(source, filter);
@ -462,21 +450,19 @@ public final class StatoolInfosCLI
}
}
}
else if (isMatching(args, "uptime", "\\s*.+\\s*"))
else if (isMatching(args, "uptime", "\\s*.+\\.conf\\s*"))
{
Chrono chrono = new Chrono().start();
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs)
File configurationFile = new File(StringUtils.trim(args[1]));
try
{
try
{
StatoolInfos.uptime(input);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
StatoolInfos.uptime(configurationFile);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
System.out.println(chrono.format());
}

67
src/fr/devinsy/statoolinfos/core/BotFilter.java Normal file
View file

@ -0,0 +1,67 @@
/*
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.core;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
/**
* The Enum LogFilter.
*/
public enum BotFilter
{
ALL,
BOT,
NOBOT;
/**
* Matches.
*
* @param log
* the log
* @return true, if successful
*/
public boolean matches(final HttpAccessLog log)
{
boolean result;
if (log == null)
{
result = false;
}
else if (this == ALL)
{
result = true;
}
else if ((this == BOT) && (log.isBot()))
{
result = true;
}
else if ((this == NOBOT) && (!log.isBot()))
{
result = true;
}
else
{
result = false;
}
//
return result;
}
}

15
src/fr/devinsy/statoolinfos/core/Configuration.java
View file

@ -331,6 +331,21 @@ public class Configuration extends PathPropertyList
return result;
}
/**
* Gets the probe http access log pattern.
*
* @return the probe http access log pattern
*/
public String getProbeHttpAccessLogPattern()
{
String result;
result = get("conf.probe.httpaccesslog.pattern");
//
return result;
}
/**
* Gets the probe http access log file.
*

24
src/fr/devinsy/statoolinfos/core/StatoolInfos.java
View file

@ -169,7 +169,7 @@ public class StatoolInfos
* the source
* @throws IOException
*/
public static void listIps(final File source, final LogFilter filter)
public static void listIps(final File source, final BotFilter filter)
{
try
{
@ -180,7 +180,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log))
{
@ -219,7 +219,7 @@ public class StatoolInfos
* @param source
* the source
*/
public static void listUserAgents(final File source, final LogFilter filter)
public static void listUserAgents(final File source, final BotFilter filter)
{
try
{
@ -230,7 +230,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log))
{
@ -270,7 +270,7 @@ public class StatoolInfos
* @param source
* the source
*/
public static void listVisitors(final File source, final LogFilter filter)
public static void listVisitors(final File source, final BotFilter filter)
{
try
{
@ -281,7 +281,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log))
{
@ -336,7 +336,7 @@ public class StatoolInfos
* @param source
* the source
*/
public static void statIps(final File source, final LogFilter filter)
public static void statIps(final File source, final BotFilter filter)
{
try
{
@ -347,7 +347,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log))
{
stator.putLog(log);
@ -387,7 +387,7 @@ public class StatoolInfos
* @param source
* the source
*/
public static void statUserAgents(final File source, final LogFilter filter)
public static void statUserAgents(final File source, final BotFilter filter)
{
try
{
@ -398,7 +398,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log))
{
stator.putLog(log);
@ -438,7 +438,7 @@ public class StatoolInfos
* @param source
* the source
*/
public static void statVisitors(final File source, final LogFilter filter)
public static void statVisitors(final File source, final BotFilter filter)
{
try
{
@ -449,7 +449,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log))
{
stator.putLog(log);

26
src/fr/devinsy/statoolinfos/metrics/Prober.java
View file

@ -75,9 +75,11 @@ public class Prober
{
logger.info("== Processing HttpAccessLog.");
String source = configuration.getProbeHttpAccessLogSource();
String patternRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("source=[{}]", source);
logger.info("pattern=[{}]", patternRegex);
PathCounters data = HttpAccessLogAnalyzer.probe(source);
PathCounters data = HttpAccessLogAnalyzer.probe(source, patternRegex);
counters.putAll(data);
}
@ -176,15 +178,19 @@ public class Prober
for (String year : years)
{
{
// Year.
PathCounter yearCounter = prefixCounters.get(prefix, year);
if (yearCounter != null)
{
String line = String.format("%s.%s=%s", yearCounter.getPath(), yearCounter.getTimeMark(), yearCounter.getCounter());
metrics.appendln(line);
}
}
// Year stat is complicated to build because needs all the
// log of one year.
// {
// // Year.
// PathCounter yearCounter = prefixCounters.get(prefix, year);
// if (yearCounter != null)
// {
// String line = String.format("%s.%s=%s",
// yearCounter.getPath(), yearCounter.getTimeMark(),
// yearCounter.getCounter());
// metrics.appendln(line);
// }
// }
{
// Months.

88
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java
View file

@ -26,13 +26,13 @@ import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator;
/**
@ -44,6 +44,12 @@ public class HttpAccessLogAnalyzer
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
// log_format combined '$remote_addr - $remote_user [$time_local] '
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
public static final Pattern COMBINED_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
private int errorCount;
private PathCounters counters;
private VisitCounters visits;
@ -124,9 +130,22 @@ public class HttpAccessLogAnalyzer
* @throws StatoolInfosException
* the statool infos exception
*/
public void probe(final File file) throws IOException
public void probe(final File file, final String patternRegex) throws IOException
{
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
//
Pattern pattern;
if (patternRegex == null)
{
pattern = COMBINED_PATTERN;
}
else
{
pattern = Pattern.compile(patternRegex);
}
//
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
@ -134,7 +153,15 @@ public class HttpAccessLogAnalyzer
try
{
probeLine(line);
HttpAccessLog log = parseLog(line, pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
probeLog(log);
}
}
catch (Exception exception)
{
@ -151,16 +178,10 @@ public class HttpAccessLogAnalyzer
* @param line
* the line
*/
public void probeLine(final String line)
public void probeLog(final HttpAccessLog log)
{
HttpAccessLog log = parseNginxCombinedLog(line);
// logger.info("==================");
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
if (log != null)
{
// logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString());
@ -384,30 +405,6 @@ public class HttpAccessLogAnalyzer
return result;
}
/**
* Parses the log.
*
* @param line
* the line
* @return the http log
*/
public static HttpAccessLog parseNginxCombinedLog(final String line)
{
HttpAccessLog result;
// log_format combined '$remote_addr - $remote_user [$time_local] '
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
String combinedPattern = "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$";
Pattern pattern = Pattern.compile(combinedPattern);
result = parseLog(line, pattern);
//
return result;
}
/**
* Probe.
*
@ -418,30 +415,15 @@ public class HttpAccessLogAnalyzer
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final String source) throws IOException, StatoolInfosException
public static PathCounters probe(final String source, final String patternRegex) throws IOException, StatoolInfosException
{
PathCounters result;
HttpAccessLogAnalyzer analyzer = new HttpAccessLogAnalyzer();
if (!StringUtils.isBlank(source))
for (File file : FilesUtils.searchByWildcard(source))
{
if (source.endsWith("*"))
{
String prefix = source.substring(0, source.length() - 1);
for (File file : new File(prefix).getParentFile().listFiles())
{
if (file.getName().startsWith(FilenameUtils.getName(prefix)))
{
analyzer.probe(file);
}
}
}
else
{
File file = new File(source);
analyzer.probe(file);
}
analyzer.probe(file, patternRegex);
}
result = analyzer.getCounters();

98
src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java
View file

@ -26,13 +26,12 @@ import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator;
/**
@ -44,6 +43,9 @@ public class HttpErrorLogAnalyzer
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
public static final Pattern NGINX_ERROR_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
public static final Pattern APACHE_ERROR_PATTERN = Pattern.compile("TODO^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
private PathCounters counters;
private int errorCount;
@ -82,20 +84,44 @@ public class HttpErrorLogAnalyzer
*/
public void probe(final File file) throws IOException
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
if ((file != null) && (!file.isFile()) || (file.exists()))
{
String line = iterator.next();
try
//
Pattern pattern;
if (LineIterator.readFirstLine(file).startsWith("["))
{
probeLine(line);
pattern = APACHE_ERROR_PATTERN;
}
catch (Exception exception)
else
{
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
exception.printStackTrace();
this.errorCount += 1;
pattern = NGINX_ERROR_PATTERN;
}
//
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
try
{
HttpErrorLog log = parseLog(line, pattern);
// logger.info("==================");
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
probeLog(log);
}
}
catch (Exception exception)
{
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
exception.printStackTrace();
this.errorCount += 1;
}
}
}
}
@ -106,29 +132,16 @@ public class HttpErrorLogAnalyzer
* @param line
* the line
*/
public void probeLine(final String line)
public void probeLog(final HttpErrorLog log)
{
HttpErrorLog log = parseNginxLog(line);
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// logger.info("==================");
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
// logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString());
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.http.hits
this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
}
// metrics.http.hits
this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
// TODO metrics.http.errors.php
}
@ -212,24 +225,9 @@ public class HttpErrorLogAnalyzer
HttpErrorLogAnalyzer analyzer = new HttpErrorLogAnalyzer();
if (!StringUtils.isBlank(source))
for (File file : FilesUtils.searchByWildcard(source))
{
if (source.endsWith("*"))
{
String prefix = source.substring(0, source.length() - 1);
for (File file : new File(prefix).getParentFile().listFiles())
{
if (file.getName().startsWith(FilenameUtils.getName(prefix)))
{
analyzer.probe(file);
}
}
}
else
{
File file = new File(source);
analyzer.probe(file);
}
analyzer.probe(file);
}
result = analyzer.getCounters();

2
src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
View file

@ -99,7 +99,7 @@ public class HttpLogIterator
try
{
String line = this.iterator.next();
result = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
result = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
}
catch (Exception exception)
{

67
src/fr/devinsy/statoolinfos/util/FilesUtils.java
View file

@ -21,6 +21,7 @@ package fr.devinsy.statoolinfos.util;
import java.io.File;
import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
/**
@ -104,6 +105,44 @@ public class FilesUtils
return result;
}
/**
* Search by wildcard.
*
* @param source
* the source
* @return the files
*/
public static Files searchByWildcard(final String source)
{
Files result;
result = new Files();
if (!StringUtils.isBlank(source))
{
if (source.endsWith("*"))
{
String prefix = source.substring(0, source.length() - 1);
String shortPrefix = FilenameUtils.getName(prefix);
for (File file : new File(prefix).getParentFile().listFiles())
{
if (file.getName().startsWith(shortPrefix))
{
result.add(file);
}
}
}
else
{
File file = new File(source);
result.add(file);
}
}
//
return result;
}
/**
* List recursively.
*
@ -131,4 +170,32 @@ public class FilesUtils
//
return result;
}
/**
* Search starting with.
*
* @param source
* the source
* @param extensions
* the extensions
* @return the files
*/
public static Files searchStartingWith(final File source, final String... extensions)
{
Files result;
result = new Files();
Files full = listRecursively(source);
for (File file : full)
{
if (StringUtils.startsWithAny(file.getName(), extensions))
{
result.add(file);
}
}
//
return result;
}
}

38
src/fr/devinsy/statoolinfos/util/LineIterator.java
View file

@ -131,4 +131,42 @@ public class LineIterator
this.ready = true;
}
}
/**
* Read first line.
*
* @param file
* the file
* @return the string
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static String readFirstLine(final File file) throws IOException
{
String result;
LineIterator iterator = null;
try
{
iterator = new LineIterator(file);
if (iterator.hasNext())
{
result = iterator.next();
}
else
{
result = null;
}
}
finally
{
if (iterator != null)
{
iterator.close();
}
}
//
return result;
}
}