statool/statoolinfos
statool/statoolinfos
1
0
Fork 0
Code Issues Pull requests Releases 8 Activity

Improved help message, custom pattern log support and Apache patterns.

Browse source
This commit is contained in:
Christian P. MOMON 2021-12-08 00:39:48 +01:00
parent 11ba05212c
commit 8a563ff194
10 changed files with 380 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java
View file

@ -25,7 +25,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.LogFilter; import fr.devinsy.statoolinfos.core.BotFilter;
import fr.devinsy.statoolinfos.core.StatoolInfos; import fr.devinsy.statoolinfos.core.StatoolInfos;
import fr.devinsy.statoolinfos.util.BuildInformation; import fr.devinsy.statoolinfos.util.BuildInformation;
import fr.devinsy.statoolinfos.util.Chrono; import fr.devinsy.statoolinfos.util.Chrono;
@ -104,20 +104,22 @@ public final class StatoolInfosCLI
message.appendln("Usage:"); message.appendln("Usage:");
message.appendln(" statoolinfos [ -h | -help | --help ]"); message.appendln(" statoolinfos [ -h | -help | --help ]");
message.appendln(" statoolinfos [ -v | -version | --version ]"); message.appendln(" statoolinfos [ -v | -version | --version ]");
message.appendln(" statoolinfos build <fileordirectory> build property files from conf and input"); message.appendln();
message.appendln(" statoolinfos clear <fileordirectory> remove property files from conf"); message.appendln(" statoolinfos build <configurationfile> build property files from conf and input");
message.appendln(" statoolinfos crawl <fileordirectory> crawl all file from conf and input"); message.appendln(" statoolinfos clear <configurationfile> remove property files from conf");
message.appendln(" statoolinfos format <fileordirectory> format in tiny way"); message.appendln(" statoolinfos crawl <configurationfile> crawl all file from conf and input");
message.appendln(" statoolinfos htmlize <fileordirectory> generate web pages from conf"); message.appendln(" statoolinfos htmlize <configurationfile> generate web pages from conf");
message.appendln(" statoolinfos probe <configurationfile> generate metrics files from conf");
message.appendln(" statoolinfos uptime <configurationfile> update uptime journal");
message.appendln();
message.appendln(" statoolinfos format <fileordirectory> format property files in tiny way");
message.appendln(" statoolinfos list ip [-bot|-nobot] <fileordirectory> generate ip list from log file"); message.appendln(" statoolinfos list ip [-bot|-nobot] <fileordirectory> generate ip list from log file");
message.appendln(" statoolinfos list ua [-bot|-nobot] <fileordirectory> generate user agent list from log file"); message.appendln(" statoolinfos list ua [-bot|-nobot] <fileordirectory> generate user agent list from log file");
message.appendln(" statoolinfos list visitors [-bot|-nobot] <fileordirectory> generate visitors (ip+ua) list from log file"); message.appendln(" statoolinfos list visitors [-bot|-nobot] <fileordirectory> generate visitors (ip+ua) list from log file");
message.appendln(" statoolinfos probe <fileordirectory> generate metrics files from conf");
message.appendln(" statoolinfos stat ip [-bot|-nobot] <fileordirectory> generate stats about ip from log file"); message.appendln(" statoolinfos stat ip [-bot|-nobot] <fileordirectory> generate stats about ip from log file");
message.appendln(" statoolinfos stat ua [-bot|-nobot] <fileordirectory> generate stats about user agent from log file"); message.appendln(" statoolinfos stat ua [-bot|-nobot] <fileordirectory> generate stats about user agent from log file");
message.appendln(" statoolinfos stat visitors [-bot|-nobot] <fileordirectory> generate stats about visitors (ip+ua) from log file"); message.appendln(" statoolinfos stat visitors [-bot|-nobot] <fileordirectory> generate stats about visitors (ip+ua) from log file");
message.appendln(" statoolinfos tagdate <fileordirectory> update the file.datetime file"); message.appendln(" statoolinfos tagdate <fileordirectory> update the file.datetime file");
message.appendln(" statoolinfos uptime <fileordirectory> update uptime journal");
System.out.println(message.toString()); System.out.println(message.toString());
} }
@ -200,21 +202,21 @@ public final class StatoolInfosCLI
* the source * the source
* @return the log filter * @return the log filter
*/ */
private static LogFilter parseLogFilterOption(final String source) private static BotFilter parseLogFilterOption(final String source)
{ {
LogFilter result; BotFilter result;
if (StringUtils.equals(source, "-all")) if (StringUtils.equals(source, "-all"))
{ {
result = LogFilter.ALL; result = BotFilter.ALL;
} }
else if (StringUtils.equals(source, "-bot")) else if (StringUtils.equals(source, "-bot"))
{ {
result = LogFilter.BOT; result = BotFilter.BOT;
} }
else if (StringUtils.equals(source, "-nobot")) else if (StringUtils.equals(source, "-nobot"))
{ {
result = LogFilter.NOBOT; result = BotFilter.NOBOT;
} }
else else
{ {
@ -271,51 +273,43 @@ public final class StatoolInfosCLI
{ {
displayVersion(); displayVersion();
} }
else if (isMatching(args, "build", "\\s*.+\\s*")) else if (isMatching(args, "build", "\\s*.+\\.conf\\s*"))
{ {
Files inputs = convertPath(StringUtils.trim(args[1])); File configurationFile = new File(StringUtils.trim(args[1]));
for (File input : inputs) try
{ {
try StatoolInfos.build(configurationFile);
{ }
StatoolInfos.build(input); catch (Exception exception)
} {
catch (Exception exception) logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
}
} }
} }
else if (isMatching(args, "clear", "\\s*.+\\s*")) else if (isMatching(args, "clear", "\\s*.+\\.conf\\s*"))
{ {
Files inputs = convertPath(StringUtils.trim(args[1])); File configurationFile = new File(StringUtils.trim(args[1]));
for (File input : inputs) try
{ {
try StatoolInfos.clear(configurationFile);
{ }
StatoolInfos.clear(input); catch (Exception exception)
} {
catch (Exception exception) logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
}
} }
} }
else if (isMatching(args, "crawl", "\\s*.+\\s*")) else if (isMatching(args, "crawl", "\\s*.+\\.conf\\s*"))
{ {
Chrono chrono = new Chrono().start(); Chrono chrono = new Chrono().start();
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs) File configurationFile = new File(StringUtils.trim(args[1]));
try
{ {
try StatoolInfos.crawl(configurationFile);
{ }
StatoolInfos.crawl(input); catch (Exception exception)
} {
catch (Exception exception) logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
{ exception.printStackTrace();
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
} }
System.out.println(chrono.format()); System.out.println(chrono.format());
} }
@ -335,21 +329,18 @@ public final class StatoolInfosCLI
} }
} }
} }
else if (isMatching(args, "htmlize", "\\s*.+\\s*")) else if (isMatching(args, "htmlize", "\\s*.+\\.conf\\s*"))
{ {
Chrono chrono = new Chrono().start(); Chrono chrono = new Chrono().start();
Files inputs = convertPath(StringUtils.trim(args[1])); File configurationFile = new File(StringUtils.trim(args[1]));
for (File input : inputs) try
{ {
try StatoolInfos.htmlize(configurationFile);
{ }
StatoolInfos.htmlize(input); catch (Exception exception)
} {
catch (Exception exception) logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
{ exception.printStackTrace();
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
} }
System.out.println(chrono.format()); System.out.println(chrono.format());
} }
@ -357,11 +348,11 @@ public final class StatoolInfosCLI
{ {
File source = new File(args[2]); File source = new File(args[2]);
StatoolInfos.listIps(source, LogFilter.ALL); StatoolInfos.listIps(source, BotFilter.ALL);
} }
else if (isMatching(args, "list", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) else if (isMatching(args, "list", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{ {
LogFilter filter = parseLogFilterOption(args[2]); BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]); File source = new File(args[3]);
StatoolInfos.listIps(source, filter); StatoolInfos.listIps(source, filter);
@ -370,11 +361,11 @@ public final class StatoolInfosCLI
{ {
File source = new File(args[2]); File source = new File(args[2]);
StatoolInfos.listUserAgents(source, LogFilter.ALL); StatoolInfos.listUserAgents(source, BotFilter.ALL);
} }
else if (isMatching(args, "list", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) else if (isMatching(args, "list", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{ {
LogFilter filter = parseLogFilterOption(args[2]); BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]); File source = new File(args[3]);
StatoolInfos.listUserAgents(source, filter); StatoolInfos.listUserAgents(source, filter);
@ -383,40 +374,37 @@ public final class StatoolInfosCLI
{ {
File source = new File(args[2]); File source = new File(args[2]);
StatoolInfos.listVisitors(source, LogFilter.ALL); StatoolInfos.listVisitors(source, BotFilter.ALL);
} }
else if (isMatching(args, "list", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) else if (isMatching(args, "list", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{ {
LogFilter filter = parseLogFilterOption(args[2]); BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]); File source = new File(args[3]);
StatoolInfos.listVisitors(source, filter); StatoolInfos.listVisitors(source, filter);
} }
else if (isMatching(args, "probe", "\\s*.+\\s*")) else if (isMatching(args, "probe", "\\s*.+\\.conf\\s*"))
{ {
Files inputs = convertPath(StringUtils.trim(args[1])); File configurationFile = new File(StringUtils.trim(args[1]));
for (File input : inputs) try
{ {
try StatoolInfos.probe(configurationFile);
{ }
StatoolInfos.probe(input); catch (Exception exception)
} {
catch (Exception exception) logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
{ exception.printStackTrace();
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
} }
} }
else if (isMatching(args, "stat", "ip", "\\s*\\S+\\s*")) else if (isMatching(args, "stat", "ip", "\\s*\\S+\\s*"))
{ {
File source = new File(args[2]); File source = new File(args[2]);
StatoolInfos.statIps(source, LogFilter.ALL); StatoolInfos.statIps(source, BotFilter.ALL);
} }
else if (isMatching(args, "stat", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) else if (isMatching(args, "stat", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{ {
LogFilter filter = parseLogFilterOption(args[2]); BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]); File source = new File(args[3]);
StatoolInfos.statIps(source, filter); StatoolInfos.statIps(source, filter);
@ -424,11 +412,11 @@ public final class StatoolInfosCLI
else if (isMatching(args, "stat", "(useragent|ua)", "\\s*\\S+\\s*")) else if (isMatching(args, "stat", "(useragent|ua)", "\\s*\\S+\\s*"))
{ {
File source = new File(args[2]); File source = new File(args[2]);
StatoolInfos.statUserAgents(source, LogFilter.ALL); StatoolInfos.statUserAgents(source, BotFilter.ALL);
} }
else if (isMatching(args, "stat", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) else if (isMatching(args, "stat", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{ {
LogFilter filter = parseLogFilterOption(args[2]); BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]); File source = new File(args[3]);
StatoolInfos.statUserAgents(source, filter); StatoolInfos.statUserAgents(source, filter);
@ -437,11 +425,11 @@ public final class StatoolInfosCLI
{ {
File source = new File(args[2]); File source = new File(args[2]);
StatoolInfos.statVisitors(source, LogFilter.ALL); StatoolInfos.statVisitors(source, BotFilter.ALL);
} }
else if (isMatching(args, "stat", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) else if (isMatching(args, "stat", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
{ {
LogFilter filter = parseLogFilterOption(args[2]); BotFilter filter = parseLogFilterOption(args[2]);
File source = new File(args[3]); File source = new File(args[3]);
StatoolInfos.statVisitors(source, filter); StatoolInfos.statVisitors(source, filter);
@ -462,21 +450,19 @@ public final class StatoolInfosCLI
} }
} }
} }
else if (isMatching(args, "uptime", "\\s*.+\\s*")) else if (isMatching(args, "uptime", "\\s*.+\\.conf\\s*"))
{ {
Chrono chrono = new Chrono().start(); Chrono chrono = new Chrono().start();
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs) File configurationFile = new File(StringUtils.trim(args[1]));
try
{ {
try StatoolInfos.uptime(configurationFile);
{ }
StatoolInfos.uptime(input); catch (Exception exception)
} {
catch (Exception exception) logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
{ exception.printStackTrace();
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
exception.printStackTrace();
}
} }
System.out.println(chrono.format()); System.out.println(chrono.format());
} }

67
src/fr/devinsy/statoolinfos/core/BotFilter.java Normal file
View file

@ -0,0 +1,67 @@
/*
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.core;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
/**
* The Enum LogFilter.
*/
public enum BotFilter
{
ALL,
BOT,
NOBOT;
/**
* Matches.
*
* @param log
* the log
* @return true, if successful
*/
public boolean matches(final HttpAccessLog log)
{
boolean result;
if (log == null)
{
result = false;
}
else if (this == ALL)
{
result = true;
}
else if ((this == BOT) && (log.isBot()))
{
result = true;
}
else if ((this == NOBOT) && (!log.isBot()))
{
result = true;
}
else
{
result = false;
}
//
return result;
}
}

15
src/fr/devinsy/statoolinfos/core/Configuration.java
View file

@ -331,6 +331,21 @@ public class Configuration extends PathPropertyList
return result; return result;
} }
/**
* Gets the probe http access log pattern.
*
* @return the probe http access log pattern
*/
public String getProbeHttpAccessLogPattern()
{
String result;
result = get("conf.probe.httpaccesslog.pattern");
//
return result;
}
/** /**
* Gets the probe http access log file. * Gets the probe http access log file.
* *

24
src/fr/devinsy/statoolinfos/core/StatoolInfos.java
View file

@ -169,7 +169,7 @@ public class StatoolInfos
* the source * the source
* @throws IOException * @throws IOException
*/ */
public static void listIps(final File source, final LogFilter filter) public static void listIps(final File source, final BotFilter filter)
{ {
try try
{ {
@ -180,7 +180,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
@ -219,7 +219,7 @@ public class StatoolInfos
* @param source * @param source
* the source * the source
*/ */
public static void listUserAgents(final File source, final LogFilter filter) public static void listUserAgents(final File source, final BotFilter filter)
{ {
try try
{ {
@ -230,7 +230,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
@ -270,7 +270,7 @@ public class StatoolInfos
* @param source * @param source
* the source * the source
*/ */
public static void listVisitors(final File source, final LogFilter filter) public static void listVisitors(final File source, final BotFilter filter)
{ {
try try
{ {
@ -281,7 +281,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
@ -336,7 +336,7 @@ public class StatoolInfos
* @param source * @param source
* the source * the source
*/ */
public static void statIps(final File source, final LogFilter filter) public static void statIps(final File source, final BotFilter filter)
{ {
try try
{ {
@ -347,7 +347,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
stator.putLog(log); stator.putLog(log);
@ -387,7 +387,7 @@ public class StatoolInfos
* @param source * @param source
* the source * the source
*/ */
public static void statUserAgents(final File source, final LogFilter filter) public static void statUserAgents(final File source, final BotFilter filter)
{ {
try try
{ {
@ -398,7 +398,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
stator.putLog(log); stator.putLog(log);
@ -438,7 +438,7 @@ public class StatoolInfos
* @param source * @param source
* the source * the source
*/ */
public static void statVisitors(final File source, final LogFilter filter) public static void statVisitors(final File source, final BotFilter filter)
{ {
try try
{ {
@ -449,7 +449,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
stator.putLog(log); stator.putLog(log);

26
src/fr/devinsy/statoolinfos/metrics/Prober.java
View file

@ -75,9 +75,11 @@ public class Prober
{ {
logger.info("== Processing HttpAccessLog."); logger.info("== Processing HttpAccessLog.");
String source = configuration.getProbeHttpAccessLogSource(); String source = configuration.getProbeHttpAccessLogSource();
String patternRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("source=[{}]", source); logger.info("source=[{}]", source);
logger.info("pattern=[{}]", patternRegex);
PathCounters data = HttpAccessLogAnalyzer.probe(source); PathCounters data = HttpAccessLogAnalyzer.probe(source, patternRegex);
counters.putAll(data); counters.putAll(data);
} }
@ -176,15 +178,19 @@ public class Prober
for (String year : years) for (String year : years)
{ {
{ // Year stat is complicated to build because needs all the
// Year. // log of one year.
PathCounter yearCounter = prefixCounters.get(prefix, year); // {
if (yearCounter != null) // // Year.
{ // PathCounter yearCounter = prefixCounters.get(prefix, year);
String line = String.format("%s.%s=%s", yearCounter.getPath(), yearCounter.getTimeMark(), yearCounter.getCounter()); // if (yearCounter != null)
metrics.appendln(line); // {
} // String line = String.format("%s.%s=%s",
} // yearCounter.getPath(), yearCounter.getTimeMark(),
// yearCounter.getCounter());
// metrics.appendln(line);
// }
// }
{ {
// Months. // Months.

88
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java
View file

@ -26,13 +26,13 @@ import java.util.Locale;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters; import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator; import fr.devinsy.statoolinfos.util.LineIterator;
/** /**
@ -44,6 +44,12 @@ public class HttpAccessLogAnalyzer
public static final String DEFAULT_CHARSET_NAME = "UTF-8"; public static final String DEFAULT_CHARSET_NAME = "UTF-8";
// log_format combined '$remote_addr - $remote_user [$time_local] '
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
public static final Pattern COMBINED_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
private int errorCount; private int errorCount;
private PathCounters counters; private PathCounters counters;
private VisitCounters visits; private VisitCounters visits;
@ -124,9 +130,22 @@ public class HttpAccessLogAnalyzer
* @throws StatoolInfosException * @throws StatoolInfosException
* the statool infos exception * the statool infos exception
*/ */
public void probe(final File file) throws IOException public void probe(final File file, final String patternRegex) throws IOException
{ {
System.out.println("Probing file [" + file.getAbsolutePath() + "]"); System.out.println("Probing file [" + file.getAbsolutePath() + "]");
//
Pattern pattern;
if (patternRegex == null)
{
pattern = COMBINED_PATTERN;
}
else
{
pattern = Pattern.compile(patternRegex);
}
//
LineIterator iterator = new LineIterator(file); LineIterator iterator = new LineIterator(file);
while (iterator.hasNext()) while (iterator.hasNext())
{ {
@ -134,7 +153,15 @@ public class HttpAccessLogAnalyzer
try try
{ {
probeLine(line); HttpAccessLog log = parseLog(line, pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
probeLog(log);
}
} }
catch (Exception exception) catch (Exception exception)
{ {
@ -151,16 +178,10 @@ public class HttpAccessLogAnalyzer
* @param line * @param line
* the line * the line
*/ */
public void probeLine(final String line) public void probeLog(final HttpAccessLog log)
{ {
HttpAccessLog log = parseNginxCombinedLog(line);
// logger.info("=================="); // logger.info("==================");
if (log == null) if (log != null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{ {
// logger.info("LINE IS MATCHING [{}]", log); // logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString()); // logger.info(log.getHttpUserAgent().toString());
@ -384,30 +405,6 @@ public class HttpAccessLogAnalyzer
return result; return result;
} }
/**
* Parses the log.
*
* @param line
* the line
* @return the http log
*/
public static HttpAccessLog parseNginxCombinedLog(final String line)
{
HttpAccessLog result;
// log_format combined '$remote_addr - $remote_user [$time_local] '
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
String combinedPattern = "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$";
Pattern pattern = Pattern.compile(combinedPattern);
result = parseLog(line, pattern);
//
return result;
}
/** /**
* Probe. * Probe.
* *
@ -418,30 +415,15 @@ public class HttpAccessLogAnalyzer
* @throws StatoolInfosException * @throws StatoolInfosException
* the statool infos exception * the statool infos exception
*/ */
public static PathCounters probe(final String source) throws IOException, StatoolInfosException public static PathCounters probe(final String source, final String patternRegex) throws IOException, StatoolInfosException
{ {
PathCounters result; PathCounters result;
HttpAccessLogAnalyzer analyzer = new HttpAccessLogAnalyzer(); HttpAccessLogAnalyzer analyzer = new HttpAccessLogAnalyzer();
if (!StringUtils.isBlank(source)) for (File file : FilesUtils.searchByWildcard(source))
{ {
if (source.endsWith("*")) analyzer.probe(file, patternRegex);
{
String prefix = source.substring(0, source.length() - 1);
for (File file : new File(prefix).getParentFile().listFiles())
{
if (file.getName().startsWith(FilenameUtils.getName(prefix)))
{
analyzer.probe(file);
}
}
}
else
{
File file = new File(source);
analyzer.probe(file);
}
} }
result = analyzer.getCounters(); result = analyzer.getCounters();

98
src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java
View file

@ -26,13 +26,12 @@ import java.util.Locale;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters; import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator; import fr.devinsy.statoolinfos.util.LineIterator;
/** /**
@ -44,6 +43,9 @@ public class HttpErrorLogAnalyzer
public static final String DEFAULT_CHARSET_NAME = "UTF-8"; public static final String DEFAULT_CHARSET_NAME = "UTF-8";
public static final Pattern NGINX_ERROR_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
public static final Pattern APACHE_ERROR_PATTERN = Pattern.compile("TODO^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
private PathCounters counters; private PathCounters counters;
private int errorCount; private int errorCount;
@ -82,20 +84,44 @@ public class HttpErrorLogAnalyzer
*/ */
public void probe(final File file) throws IOException public void probe(final File file) throws IOException
{ {
LineIterator iterator = new LineIterator(file); if ((file != null) && (!file.isFile()) || (file.exists()))
while (iterator.hasNext())
{ {
String line = iterator.next(); //
Pattern pattern;
try if (LineIterator.readFirstLine(file).startsWith("["))
{ {
probeLine(line); pattern = APACHE_ERROR_PATTERN;
} }
catch (Exception exception) else
{ {
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage()); pattern = NGINX_ERROR_PATTERN;
exception.printStackTrace(); }
this.errorCount += 1;
//
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
try
{
HttpErrorLog log = parseLog(line, pattern);
// logger.info("==================");
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
probeLog(log);
}
}
catch (Exception exception)
{
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
exception.printStackTrace();
this.errorCount += 1;
}
} }
} }
} }
@ -106,29 +132,16 @@ public class HttpErrorLogAnalyzer
* @param line * @param line
* the line * the line
*/ */
public void probeLine(final String line) public void probeLog(final HttpErrorLog log)
{ {
HttpErrorLog log = parseNginxLog(line); // General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// logger.info("=================="); // metrics.http.hits
if (log == null) this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
// logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString());
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.http.hits
this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
}
// TODO metrics.http.errors.php // TODO metrics.http.errors.php
} }
@ -212,24 +225,9 @@ public class HttpErrorLogAnalyzer
HttpErrorLogAnalyzer analyzer = new HttpErrorLogAnalyzer(); HttpErrorLogAnalyzer analyzer = new HttpErrorLogAnalyzer();
if (!StringUtils.isBlank(source)) for (File file : FilesUtils.searchByWildcard(source))
{ {
if (source.endsWith("*")) analyzer.probe(file);
{
String prefix = source.substring(0, source.length() - 1);
for (File file : new File(prefix).getParentFile().listFiles())
{
if (file.getName().startsWith(FilenameUtils.getName(prefix)))
{
analyzer.probe(file);
}
}
}
else
{
File file = new File(source);
analyzer.probe(file);
}
} }
result = analyzer.getCounters(); result = analyzer.getCounters();

2
src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
View file

@ -99,7 +99,7 @@ public class HttpLogIterator
try try
{ {
String line = this.iterator.next(); String line = this.iterator.next();
result = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); result = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
} }
catch (Exception exception) catch (Exception exception)
{ {

67
src/fr/devinsy/statoolinfos/util/FilesUtils.java
View file

@ -21,6 +21,7 @@ package fr.devinsy.statoolinfos.util;
import java.io.File; import java.io.File;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
/** /**
@ -104,6 +105,44 @@ public class FilesUtils
return result; return result;
} }
/**
* Search by wildcard.
*
* @param source
* the source
* @return the files
*/
public static Files searchByWildcard(final String source)
{
Files result;
result = new Files();
if (!StringUtils.isBlank(source))
{
if (source.endsWith("*"))
{
String prefix = source.substring(0, source.length() - 1);
String shortPrefix = FilenameUtils.getName(prefix);
for (File file : new File(prefix).getParentFile().listFiles())
{
if (file.getName().startsWith(shortPrefix))
{
result.add(file);
}
}
}
else
{
File file = new File(source);
result.add(file);
}
}
//
return result;
}
/** /**
* List recursively. * List recursively.
* *
@ -131,4 +170,32 @@ public class FilesUtils
// //
return result; return result;
} }
/**
* Search starting with.
*
* @param source
* the source
* @param extensions
* the extensions
* @return the files
*/
public static Files searchStartingWith(final File source, final String... extensions)
{
Files result;
result = new Files();
Files full = listRecursively(source);
for (File file : full)
{
if (StringUtils.startsWithAny(file.getName(), extensions))
{
result.add(file);
}
}
//
return result;
}
} }

38
src/fr/devinsy/statoolinfos/util/LineIterator.java
View file

@ -131,4 +131,42 @@ public class LineIterator
this.ready = true; this.ready = true;
} }
} }
/**
* Read first line.
*
* @param file
* the file
* @return the string
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static String readFirstLine(final File file) throws IOException
{
String result;
LineIterator iterator = null;
try
{
iterator = new LineIterator(file);
if (iterator.hasNext())
{
result = iterator.next();
}
else
{
result = null;
}
}
finally
{
if (iterator != null)
{
iterator.close();
}
}
//
return result;
}
} }