Compare commits
No commits in common. "a9e817f653fe7a0c26e0ad501b2ab40427c9a114" and "e4236d26c374cd871224bb94d34ae557bcd67272" have entirely different histories.
a9e817f653
...
e4236d26c3
16 changed files with 289 additions and 829 deletions
|
|
@ -1,3 +1,3 @@
|
|||
LANGUAGE=fr_FR.UTF8
|
||||
LC_ALL=fr_FR.UTF-8
|
||||
/5 * * * * root /srv/statoolinfos/bin/statoolinfo.sh >> /srv/statoolinfos/statoolinfos-cron.log
|
||||
/5 * * * * root /srv/statoolinfos/bin/statoolinfo.sh > /srv/statoolinfos/statoolinfos-cron.log
|
||||
|
|
|
|||
|
|
@ -22,11 +22,10 @@ import java.io.File;
|
|||
import java.time.LocalDateTime;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.BotFilter;
|
||||
import fr.devinsy.statoolinfos.core.LogFilter;
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfos;
|
||||
import fr.devinsy.statoolinfos.util.BuildInformation;
|
||||
import fr.devinsy.statoolinfos.util.Chrono;
|
||||
|
|
@ -105,25 +104,22 @@ public final class StatoolInfosCLI
|
|||
message.appendln("Usage:");
|
||||
message.appendln(" statoolinfos [ -h | -help | --help ]");
|
||||
message.appendln(" statoolinfos [ -v | -version | --version ]");
|
||||
message.appendln();
|
||||
message.appendln(" statoolinfos build <configurationfile> build property files from conf and input");
|
||||
message.appendln(" statoolinfos clear <configurationfile> remove property files from conf");
|
||||
message.appendln(" statoolinfos crawl <configurationfile> crawl all file from conf and input");
|
||||
message.appendln(" statoolinfos htmlize <configurationfile> generate web pages from conf");
|
||||
message.appendln(" statoolinfos probe OPTION [<directory>|<configurationfile>] generate metrics files from conf");
|
||||
message.appendln(" OPTION = [-full|-today|-previousday|-NN] with NN a day count");
|
||||
message.appendln(" statoolinfos uptime <configurationfile> update uptime journal");
|
||||
message.appendln();
|
||||
message.appendln(" statoolinfos format <fileordirectory> format property files in tiny way");
|
||||
message.appendln(" statoolinfos build <fileordirectory> build property files from conf and input");
|
||||
message.appendln(" statoolinfos clear <fileordirectory> remove property files from conf");
|
||||
message.appendln(" statoolinfos crawl <fileordirectory> crawl all file from conf and input");
|
||||
message.appendln(" statoolinfos format <fileordirectory> format in tiny way");
|
||||
message.appendln(" statoolinfos htmlize <fileordirectory> generate web pages from conf");
|
||||
message.appendln(" statoolinfos list ip [-bot|-nobot] <fileordirectory> generate ip list from log file");
|
||||
message.appendln(" statoolinfos list ua [-bot|-nobot] <fileordirectory> generate user agent list from log file");
|
||||
message.appendln(" statoolinfos list visitors [-bot|-nobot] <fileordirectory> generate visitors (ip+ua) list from log file");
|
||||
message.appendln(" statoolinfos probe <fileordirectory> generate metrics files from conf");
|
||||
message.appendln(" statoolinfos stat ip [-bot|-nobot] <fileordirectory> generate stats about ip from log file");
|
||||
message.appendln(" statoolinfos stat ua [-bot|-nobot] <fileordirectory> generate stats about user agent from log file");
|
||||
message.appendln(" statoolinfos stat visitors [-bot|-nobot] <fileordirectory> generate stats about visitors (ip+ua) from log file");
|
||||
message.appendln(" statoolinfos tagdate <fileordirectory> update the file.datetime file");
|
||||
message.appendln(" statoolinfos uptime <fileordirectory> update uptime journal");
|
||||
|
||||
System.out.print(message.toString());
|
||||
System.out.println(message.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -135,7 +131,7 @@ public final class StatoolInfosCLI
|
|||
|
||||
message.appendln(BuildInformation.instance().version());
|
||||
|
||||
System.out.print(message.toString());
|
||||
System.out.println(message.toString());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -204,21 +200,21 @@ public final class StatoolInfosCLI
|
|||
* the source
|
||||
* @return the log filter
|
||||
*/
|
||||
private static BotFilter parseLogFilterOption(final String source)
|
||||
private static LogFilter parseLogFilterOption(final String source)
|
||||
{
|
||||
BotFilter result;
|
||||
LogFilter result;
|
||||
|
||||
if (StringUtils.equals(source, "-all"))
|
||||
{
|
||||
result = BotFilter.ALL;
|
||||
result = LogFilter.ALL;
|
||||
}
|
||||
else if (StringUtils.equals(source, "-bot"))
|
||||
{
|
||||
result = BotFilter.BOT;
|
||||
result = LogFilter.BOT;
|
||||
}
|
||||
else if (StringUtils.equals(source, "-nobot"))
|
||||
{
|
||||
result = BotFilter.NOBOT;
|
||||
result = LogFilter.NOBOT;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -275,44 +271,52 @@ public final class StatoolInfosCLI
|
|||
{
|
||||
displayVersion();
|
||||
}
|
||||
else if (isMatching(args, "build", "\\s*.+\\.conf\\s*"))
|
||||
else if (isMatching(args, "build", "\\s*.+\\s*"))
|
||||
{
|
||||
Files inputs = convertPath(StringUtils.trim(args[1]));
|
||||
for (File input : inputs)
|
||||
{
|
||||
File configurationFile = new File(StringUtils.trim(args[1]));
|
||||
try
|
||||
{
|
||||
StatoolInfos.build(configurationFile);
|
||||
StatoolInfos.build(input);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
|
||||
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
|
||||
}
|
||||
}
|
||||
else if (isMatching(args, "clear", "\\s*.+\\.conf\\s*"))
|
||||
}
|
||||
else if (isMatching(args, "clear", "\\s*.+\\s*"))
|
||||
{
|
||||
Files inputs = convertPath(StringUtils.trim(args[1]));
|
||||
for (File input : inputs)
|
||||
{
|
||||
File configurationFile = new File(StringUtils.trim(args[1]));
|
||||
try
|
||||
{
|
||||
StatoolInfos.clear(configurationFile);
|
||||
StatoolInfos.clear(input);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
|
||||
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
|
||||
}
|
||||
}
|
||||
else if (isMatching(args, "crawl", "\\s*.+\\.conf\\s*"))
|
||||
}
|
||||
else if (isMatching(args, "crawl", "\\s*.+\\s*"))
|
||||
{
|
||||
Chrono chrono = new Chrono().start();
|
||||
|
||||
File configurationFile = new File(StringUtils.trim(args[1]));
|
||||
Files inputs = convertPath(StringUtils.trim(args[1]));
|
||||
for (File input : inputs)
|
||||
{
|
||||
try
|
||||
{
|
||||
StatoolInfos.crawl(configurationFile);
|
||||
StatoolInfos.crawl(input);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
|
||||
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
System.out.println(chrono.format());
|
||||
}
|
||||
else if (isMatching(args, "format", "\\s*.+\\s*"))
|
||||
|
|
@ -331,30 +335,33 @@ public final class StatoolInfosCLI
|
|||
}
|
||||
}
|
||||
}
|
||||
else if (isMatching(args, "htmlize", "\\s*.+\\.conf\\s*"))
|
||||
else if (isMatching(args, "htmlize", "\\s*.+\\s*"))
|
||||
{
|
||||
Chrono chrono = new Chrono().start();
|
||||
File configurationFile = new File(StringUtils.trim(args[1]));
|
||||
Files inputs = convertPath(StringUtils.trim(args[1]));
|
||||
for (File input : inputs)
|
||||
{
|
||||
try
|
||||
{
|
||||
StatoolInfos.htmlize(configurationFile);
|
||||
StatoolInfos.htmlize(input);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
|
||||
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
System.out.println(chrono.format());
|
||||
}
|
||||
else if (isMatching(args, "list", "ip", "\\s*\\S+\\s*"))
|
||||
{
|
||||
File source = new File(args[2]);
|
||||
|
||||
StatoolInfos.listIps(source, BotFilter.ALL);
|
||||
StatoolInfos.listIps(source, LogFilter.ALL);
|
||||
}
|
||||
else if (isMatching(args, "list", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
|
||||
{
|
||||
BotFilter filter = parseLogFilterOption(args[2]);
|
||||
LogFilter filter = parseLogFilterOption(args[2]);
|
||||
File source = new File(args[3]);
|
||||
|
||||
StatoolInfos.listIps(source, filter);
|
||||
|
|
@ -363,11 +370,11 @@ public final class StatoolInfosCLI
|
|||
{
|
||||
File source = new File(args[2]);
|
||||
|
||||
StatoolInfos.listUserAgents(source, BotFilter.ALL);
|
||||
StatoolInfos.listUserAgents(source, LogFilter.ALL);
|
||||
}
|
||||
else if (isMatching(args, "list", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
|
||||
{
|
||||
BotFilter filter = parseLogFilterOption(args[2]);
|
||||
LogFilter filter = parseLogFilterOption(args[2]);
|
||||
File source = new File(args[3]);
|
||||
|
||||
StatoolInfos.listUserAgents(source, filter);
|
||||
|
|
@ -376,53 +383,23 @@ public final class StatoolInfosCLI
|
|||
{
|
||||
File source = new File(args[2]);
|
||||
|
||||
StatoolInfos.listVisitors(source, BotFilter.ALL);
|
||||
StatoolInfos.listVisitors(source, LogFilter.ALL);
|
||||
}
|
||||
else if (isMatching(args, "list", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
|
||||
{
|
||||
BotFilter filter = parseLogFilterOption(args[2]);
|
||||
LogFilter filter = parseLogFilterOption(args[2]);
|
||||
File source = new File(args[3]);
|
||||
|
||||
StatoolInfos.listVisitors(source, filter);
|
||||
}
|
||||
else if (isMatching(args, "probe", "\\S*(-full|-today|-previousday|-\\d+)\\s*", "\\s*.+\\s*"))
|
||||
{
|
||||
File configurationFile = new File(StringUtils.trim(args[2]));
|
||||
|
||||
String filter = StringUtils.trim(args[1]);
|
||||
int dayCountFilter;
|
||||
if (filter.equals("-full"))
|
||||
{
|
||||
dayCountFilter = -1;
|
||||
}
|
||||
else if (filter.equals("-today"))
|
||||
{
|
||||
dayCountFilter = 0;
|
||||
}
|
||||
else if (filter.equals("-previousday"))
|
||||
{
|
||||
dayCountFilter = 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
dayCountFilter = -1 * NumberUtils.toInt(filter);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Files inputs = FilesUtils.searchEndingWith(configurationFile, ".conf");
|
||||
if (inputs.isEmpty())
|
||||
{
|
||||
System.out.println("No configuration file found.");
|
||||
}
|
||||
else
|
||||
else if (isMatching(args, "probe", "\\s*.+\\s*"))
|
||||
{
|
||||
Files inputs = convertPath(StringUtils.trim(args[1]));
|
||||
for (File input : inputs)
|
||||
{
|
||||
try
|
||||
{
|
||||
System.out.println("Probing [" + input + "] with day count filter " + dayCountFilter);
|
||||
StatoolInfos.probe(input, dayCountFilter);
|
||||
StatoolInfos.probe(input);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
|
|
@ -431,22 +408,15 @@ public final class StatoolInfosCLI
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
else if (isMatching(args, "stat", "ip", "\\s*\\S+\\s*"))
|
||||
{
|
||||
File source = new File(args[2]);
|
||||
|
||||
StatoolInfos.statIps(source, BotFilter.ALL);
|
||||
StatoolInfos.statIps(source, LogFilter.ALL);
|
||||
}
|
||||
else if (isMatching(args, "stat", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
|
||||
{
|
||||
BotFilter filter = parseLogFilterOption(args[2]);
|
||||
LogFilter filter = parseLogFilterOption(args[2]);
|
||||
File source = new File(args[3]);
|
||||
|
||||
StatoolInfos.statIps(source, filter);
|
||||
|
|
@ -454,11 +424,11 @@ public final class StatoolInfosCLI
|
|||
else if (isMatching(args, "stat", "(useragent|ua)", "\\s*\\S+\\s*"))
|
||||
{
|
||||
File source = new File(args[2]);
|
||||
StatoolInfos.statUserAgents(source, BotFilter.ALL);
|
||||
StatoolInfos.statUserAgents(source, LogFilter.ALL);
|
||||
}
|
||||
else if (isMatching(args, "stat", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
|
||||
{
|
||||
BotFilter filter = parseLogFilterOption(args[2]);
|
||||
LogFilter filter = parseLogFilterOption(args[2]);
|
||||
File source = new File(args[3]);
|
||||
|
||||
StatoolInfos.statUserAgents(source, filter);
|
||||
|
|
@ -467,11 +437,11 @@ public final class StatoolInfosCLI
|
|||
{
|
||||
File source = new File(args[2]);
|
||||
|
||||
StatoolInfos.statVisitors(source, BotFilter.ALL);
|
||||
StatoolInfos.statVisitors(source, LogFilter.ALL);
|
||||
}
|
||||
else if (isMatching(args, "stat", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
|
||||
{
|
||||
BotFilter filter = parseLogFilterOption(args[2]);
|
||||
LogFilter filter = parseLogFilterOption(args[2]);
|
||||
File source = new File(args[3]);
|
||||
|
||||
StatoolInfos.statVisitors(source, filter);
|
||||
|
|
@ -492,20 +462,22 @@ public final class StatoolInfosCLI
|
|||
}
|
||||
}
|
||||
}
|
||||
else if (isMatching(args, "uptime", "\\s*.+\\.conf\\s*"))
|
||||
else if (isMatching(args, "uptime", "\\s*.+\\s*"))
|
||||
{
|
||||
Chrono chrono = new Chrono().start();
|
||||
|
||||
File configurationFile = new File(StringUtils.trim(args[1]));
|
||||
Files inputs = convertPath(StringUtils.trim(args[1]));
|
||||
for (File input : inputs)
|
||||
{
|
||||
try
|
||||
{
|
||||
StatoolInfos.uptime(configurationFile);
|
||||
StatoolInfos.uptime(input);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
|
||||
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
System.out.println(chrono.format());
|
||||
}
|
||||
else
|
||||
|
|
@ -513,5 +485,8 @@ public final class StatoolInfosCLI
|
|||
System.out.println("Bad usage.");
|
||||
displayHelp();
|
||||
}
|
||||
|
||||
//
|
||||
logger.info("Done.");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -331,21 +331,6 @@ public class Configuration extends PathPropertyList
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the probe http access log pattern.
|
||||
*
|
||||
* @return the probe http access log pattern
|
||||
*/
|
||||
public String getProbeHttpAccessLogPattern()
|
||||
{
|
||||
String result;
|
||||
|
||||
result = get("conf.probe.httpaccesslog.pattern");
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the probe http access log file.
|
||||
*
|
||||
|
|
@ -497,4 +482,5 @@ public class Configuration extends PathPropertyList
|
|||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
|
||||
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
|
|
@ -23,7 +23,7 @@ import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
|
|||
/**
|
||||
* The Enum LogFilter.
|
||||
*/
|
||||
public enum BotFilter
|
||||
public enum LogFilter
|
||||
{
|
||||
ALL,
|
||||
BOT,
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.core;
|
||||
|
||||
/**
|
||||
* The Enum LogFilter.
|
||||
*/
|
||||
public enum ProbePeriod
|
||||
{
|
||||
FULL,
|
||||
TODAY,
|
||||
PREVIOUS_DAY,
|
||||
PREVIOUS_DAYS_7;
|
||||
}
|
||||
|
|
@ -169,7 +169,7 @@ public class StatoolInfos
|
|||
* the source
|
||||
* @throws IOException
|
||||
*/
|
||||
public static void listIps(final File source, final BotFilter filter)
|
||||
public static void listIps(final File source, final LogFilter filter)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
|
@ -180,7 +180,7 @@ public class StatoolInfos
|
|||
while (iterator.hasNext())
|
||||
{
|
||||
String line = iterator.next();
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
|
||||
|
||||
if (filter.matches(log))
|
||||
{
|
||||
|
|
@ -219,7 +219,7 @@ public class StatoolInfos
|
|||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public static void listUserAgents(final File source, final BotFilter filter)
|
||||
public static void listUserAgents(final File source, final LogFilter filter)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
|
@ -230,7 +230,7 @@ public class StatoolInfos
|
|||
while (iterator.hasNext())
|
||||
{
|
||||
String line = iterator.next();
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
|
||||
|
||||
if (filter.matches(log))
|
||||
{
|
||||
|
|
@ -270,7 +270,7 @@ public class StatoolInfos
|
|||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public static void listVisitors(final File source, final BotFilter filter)
|
||||
public static void listVisitors(final File source, final LogFilter filter)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
|
@ -281,7 +281,7 @@ public class StatoolInfos
|
|||
while (iterator.hasNext())
|
||||
{
|
||||
String line = iterator.next();
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
|
||||
|
||||
if (filter.matches(log))
|
||||
{
|
||||
|
|
@ -325,9 +325,9 @@ public class StatoolInfos
|
|||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public static void probe(final File configurationFile, final int dayCountFilter) throws StatoolInfosException, IOException
|
||||
public static void probe(final File configurationFile) throws StatoolInfosException, IOException
|
||||
{
|
||||
Prober.probe(configurationFile, dayCountFilter);
|
||||
Prober.probe(configurationFile);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -336,7 +336,7 @@ public class StatoolInfos
|
|||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public static void statIps(final File source, final BotFilter filter)
|
||||
public static void statIps(final File source, final LogFilter filter)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
|
@ -347,7 +347,7 @@ public class StatoolInfos
|
|||
while (iterator.hasNext())
|
||||
{
|
||||
String line = iterator.next();
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
|
||||
if (filter.matches(log))
|
||||
{
|
||||
stator.putLog(log);
|
||||
|
|
@ -387,7 +387,7 @@ public class StatoolInfos
|
|||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public static void statUserAgents(final File source, final BotFilter filter)
|
||||
public static void statUserAgents(final File source, final LogFilter filter)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
|
@ -398,7 +398,7 @@ public class StatoolInfos
|
|||
while (iterator.hasNext())
|
||||
{
|
||||
String line = iterator.next();
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
|
||||
if (filter.matches(log))
|
||||
{
|
||||
stator.putLog(log);
|
||||
|
|
@ -438,7 +438,7 @@ public class StatoolInfos
|
|||
* @param source
|
||||
* the source
|
||||
*/
|
||||
public static void statVisitors(final File source, final BotFilter filter)
|
||||
public static void statVisitors(final File source, final LogFilter filter)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
|
@ -449,7 +449,7 @@ public class StatoolInfos
|
|||
while (iterator.hasNext())
|
||||
{
|
||||
String line = iterator.next();
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
|
||||
HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
|
||||
if (filter.matches(log))
|
||||
{
|
||||
stator.putLog(log);
|
||||
|
|
|
|||
|
|
@ -748,7 +748,7 @@ public class ChartHtmlizer
|
|||
{
|
||||
for (YearWeek timestamp = startTarget; !timestamp.isAfter(endTarget); timestamp = timestamp.plusWeeks(1))
|
||||
{
|
||||
LocalDate date = timestamp.atDay(DayOfWeek.MONDAY).plusDays(6);
|
||||
LocalDate date = timestamp.atDay(DayOfWeek.MONDAY).plusWeeks(1);
|
||||
String timestampLabel = date.format(DateTimeFormatter.ofPattern("yyyy-MMM-dd", Locale.FRANCE));
|
||||
chart.getLabels().add(timestampLabel);
|
||||
|
||||
|
|
|
|||
|
|
@ -27,11 +27,7 @@
|
|||
<img src="status-warning.png" title="" style="height: 16px; width: 10px;" />Erreur rétablie<br/>
|
||||
<img src="status-alert.png" title="" style="height: 16px; width: 10px;" />Erreur en cours<br/>
|
||||
<img src="status-error.png" title="" style="height: 16px; width: 10px;" />Erreurs continues<br/>
|
||||
<img src="status-void.png" title="" style="height: 16px; width: 10px;" />Absence de donnée<br/>
|
||||
<img src="dialog-warning.svg" title="" /> =
|
||||
<img src="status-alert.png" title="" style="height: 16px; width: 10px;" /> +
|
||||
<img src="status-error.png" title="" style="height: 16px; width: 10px;" /> +
|
||||
<img src="status-void.png" title="" style="height: 16px; width: 10px;" /><br/>
|
||||
<img src="status-void.png" title="" style="height: 16px; width: 10px;" />Absence de données<br/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -39,12 +35,12 @@
|
|||
<div class="uptimeCounts">
|
||||
<ul>
|
||||
<li>Actuellement : </li>
|
||||
<li><img src="status-ok.png" title="Nominaux" /><span id="okCount">n/a</span></li>
|
||||
<li><img src="status-warning.png" title="Erreurs rétablies" /><span id="warningCount">n/a</span></li>
|
||||
<li><img src="status-alert.png" title="Erreurs en cours" /><span id="alertCount">n/a</span></li>
|
||||
<li><img src="status-error.png" title="Erreurs continues" /><span id="errorCount">n/a</span></li>
|
||||
<li><img src="status-void.png" title="Absences de données" /><span id="voidCount">n/a</span></li>
|
||||
<li><img src="dialog-warning.svg" title="Alertes" /><span id="unavailableCount">n/a</span></li>
|
||||
<li><img src="status-ok.png" title="" /><span id="okCount">n/a</span></li>
|
||||
<li><img src="status-warning.png" title="" /><span id="warningCount">n/a</span></li>
|
||||
<li><img src="status-alert.png" title="" /><span id="alertCount">n/a</span></li>
|
||||
<li><img src="status-error.png" title="" /><span id="errorCount">n/a</span></li>
|
||||
<li><img src="status-void.png" title="" /><span id="voidCount">n/a</span></li>
|
||||
<li><img src="dialog-warning.svg" title="" /><span id="unavailableCount">n/a</span></li>
|
||||
</ul>
|
||||
</div>
|
||||
<table id="uptimeTable" class="center_table table_classic left uptimeTable">
|
||||
|
|
|
|||
|
|
@ -19,13 +19,11 @@
|
|||
package fr.devinsy.statoolinfos.metrics;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.time.YearMonth;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.HashMap;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.threeten.extra.YearWeek;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
|
||||
import fr.devinsy.strings.StringList;
|
||||
|
|
@ -392,94 +390,4 @@ public class PathCounters extends HashMap<String, PathCounter>
|
|||
put(computeKey(counter.getPath(), counter.getTimeMark()), counter);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search by date.
|
||||
*
|
||||
* @param day
|
||||
* the day
|
||||
* @return the path counters
|
||||
*/
|
||||
public PathCounters searchByDate(final LocalDate date)
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
result = new PathCounters();
|
||||
|
||||
if (date != null)
|
||||
{
|
||||
String day = date.toString();
|
||||
String week = YearWeek.from(date).toString();
|
||||
String month = YearMonth.from(date).toString();
|
||||
|
||||
for (PathCounter counter : this.values())
|
||||
{
|
||||
TimeMark mark = new TimeMark(counter.getTimeMark());
|
||||
|
||||
if (mark.isDate() && (counter.getTimeMark().equals(day)))
|
||||
{
|
||||
result.put(counter);
|
||||
}
|
||||
else if (mark.isYearWeek() && (counter.getTimeMark().equals(week)))
|
||||
{
|
||||
result.put(counter);
|
||||
}
|
||||
else if (mark.isYearMonth() && (counter.getTimeMark().equals(month)))
|
||||
{
|
||||
result.put(counter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
public PathCounters searchByPeriod(final LocalDate startDate, final LocalDate endDate)
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
result = new PathCounters();
|
||||
|
||||
if ((startDate != null) && (endDate != null))
|
||||
{
|
||||
if (startDate.isAfter(endDate))
|
||||
{
|
||||
result = searchByPeriod(endDate, startDate);
|
||||
}
|
||||
else
|
||||
{
|
||||
StringSet days = new StringSet();
|
||||
StringSet weeks = new StringSet();
|
||||
StringSet months = new StringSet();
|
||||
for (LocalDate current = startDate; !current.isAfter(endDate); current = current.plusDays(1))
|
||||
{
|
||||
days.add(current.toString());
|
||||
weeks.add(YearWeek.from(current).toString());
|
||||
months.add(YearMonth.from(current).toString());
|
||||
}
|
||||
|
||||
for (PathCounter counter : this.values())
|
||||
{
|
||||
TimeMark mark = new TimeMark(counter.getTimeMark());
|
||||
|
||||
if (mark.isDate() && (days.contains(counter.getTimeMark())))
|
||||
{
|
||||
result.put(counter);
|
||||
}
|
||||
else if (mark.isYearWeek() && (weeks.contains(counter.getTimeMark())))
|
||||
{
|
||||
result.put(counter);
|
||||
}
|
||||
else if (mark.isYearMonth() && (months.contains(counter.getTimeMark())))
|
||||
{
|
||||
result.put(counter);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,26 +22,17 @@ import java.io.File;
|
|||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.Configuration;
|
||||
import fr.devinsy.statoolinfos.core.Factory;
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
|
||||
import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer;
|
||||
import fr.devinsy.statoolinfos.properties.PathProperties;
|
||||
import fr.devinsy.statoolinfos.properties.PathProperty;
|
||||
import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
|
||||
import fr.devinsy.strings.StringList;
|
||||
import fr.devinsy.strings.StringsUtils;
|
||||
|
||||
|
|
@ -54,8 +45,6 @@ public class Prober
|
|||
|
||||
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
|
||||
|
||||
public static final Pattern YEAR_PATTERN = Pattern.compile("^\\d{4}$");
|
||||
|
||||
/**
|
||||
* Instantiates a new prober.
|
||||
*/
|
||||
|
|
@ -63,44 +52,22 @@ public class Prober
|
|||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract year.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the string
|
||||
*/
|
||||
private static boolean isYear(final String value)
|
||||
{
|
||||
boolean result;
|
||||
|
||||
result = YEAR_PATTERN.matcher(value).matches();
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param configuration
|
||||
* the configuration
|
||||
* @param dayCountFilter
|
||||
* the day count filter, 0=today, 1=previous day, 7=previous
|
||||
* week…
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public static void probe(final Configuration configuration, final int dayCountFilter) throws IOException, StatoolInfosException
|
||||
public static void probe(final Configuration configuration) throws IOException, StatoolInfosException
|
||||
{
|
||||
StringList types = configuration.getProbeTypes();
|
||||
|
||||
System.out.println("Targets=" + types.toStringWithBrackets());
|
||||
|
||||
if (!types.isEmpty())
|
||||
{
|
||||
PathCounters counters = new PathCounters();
|
||||
|
||||
//
|
||||
|
|
@ -108,11 +75,9 @@ public class Prober
|
|||
{
|
||||
logger.info("== Processing HttpAccessLog.");
|
||||
String source = configuration.getProbeHttpAccessLogSource();
|
||||
String patternRegex = configuration.getProbeHttpAccessLogPattern();
|
||||
logger.info("source=[{}]", source);
|
||||
logger.info("pattern=[{}]", patternRegex);
|
||||
|
||||
PathCounters data = HttpAccessLogAnalyzer.probe(source, patternRegex);
|
||||
PathCounters data = HttpAccessLogAnalyzer.probe(source);
|
||||
counters.putAll(data);
|
||||
}
|
||||
|
||||
|
|
@ -139,24 +104,7 @@ public class Prober
|
|||
}
|
||||
|
||||
//
|
||||
if (types.containsAnyIgnoreCase("Mumble"))
|
||||
{
|
||||
logger.info("== Processing Mumble.");
|
||||
String source = configuration.getProbeHttpErrorLogSource();
|
||||
logger.info("source=[{}]", source);
|
||||
|
||||
// PathCounters data = HttpErrorLogAnalyzer.probe(source);
|
||||
// counters.putAll(data);
|
||||
}
|
||||
|
||||
// Filter.
|
||||
logger.info("== Filtering with {}", dayCountFilter);
|
||||
if (dayCountFilter >= 0)
|
||||
{
|
||||
counters = counters.searchByPeriod(LocalDate.now().minusDays(dayCountFilter), LocalDate.now());
|
||||
}
|
||||
|
||||
//
|
||||
logger.info("== Writing.");
|
||||
File target = configuration.getProbeTarget();
|
||||
logger.info("target=[{}]", target);
|
||||
|
||||
|
|
@ -166,46 +114,13 @@ public class Prober
|
|||
}
|
||||
else
|
||||
{
|
||||
if (target.exists())
|
||||
{
|
||||
// Load.
|
||||
logger.info("== Reading previous target file.");
|
||||
PathCounters previousCounters = readMetrics(target);
|
||||
logger.info("previous size={}", previousCounters.size());
|
||||
|
||||
// Merge.
|
||||
logger.info("== Merging");
|
||||
for (PathCounter counter : counters.values())
|
||||
{
|
||||
PathCounter previousCounter = previousCounters.get(counter.getPath(), counter.getTimeMark());
|
||||
if (previousCounter == null)
|
||||
{
|
||||
previousCounters.put(counter);
|
||||
}
|
||||
else
|
||||
{
|
||||
previousCounter.setCounter(counter.getCounter());
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
counters = previousCounters;
|
||||
|
||||
//
|
||||
logger.info("== Backing previous target file.");
|
||||
target.renameTo(new File(target.getParentFile(), target.getName() + ".bak"));
|
||||
}
|
||||
|
||||
writeMetrics(target, counters);
|
||||
}
|
||||
|
||||
//
|
||||
logger.info("== Writing.");
|
||||
logger.info("size={}", counters.size());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Stat.
|
||||
*
|
||||
|
|
@ -214,104 +129,13 @@ public class Prober
|
|||
* @throws IOException
|
||||
* @throws StatoolInfosException
|
||||
*/
|
||||
public static void probe(final File configurationFile, final int dayCountFilter) throws StatoolInfosException, IOException
|
||||
public static void probe(final File configurationFile) throws StatoolInfosException, IOException
|
||||
{
|
||||
logger.info("Probe {}", configurationFile.getAbsolutePath());
|
||||
|
||||
Configuration configuration = Factory.loadConfiguration(configurationFile);
|
||||
|
||||
probe(configuration, dayCountFilter);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read metrics.
|
||||
*
|
||||
* @param inputFile
|
||||
* the input file
|
||||
* @return the path counters
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public static PathCounters readMetrics(final File inputFile) throws IOException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
result = new PathCounters();
|
||||
|
||||
PathProperties lines = PathPropertyUtils.load(inputFile);
|
||||
|
||||
for (PathProperty property : lines)
|
||||
{
|
||||
String metricName = property.getMetricName();
|
||||
if (metricName != null)
|
||||
{
|
||||
String leaf = property.getLeaf();
|
||||
if (isYear(leaf))
|
||||
{
|
||||
PathCounter counter = new PathCounter(metricName, leaf);
|
||||
counter.setCounter(NumberUtils.createLong(property.getValue()));
|
||||
result.put(counter);
|
||||
}
|
||||
else if (StringUtils.equals(leaf, "months"))
|
||||
{
|
||||
String year = property.getMetricYear();
|
||||
|
||||
StringList values = StatoolInfosUtils.splitMonthValues(property.getValue());
|
||||
int monthIndex = 1;
|
||||
for (String value : values)
|
||||
{
|
||||
if (!StringUtils.isBlank(value))
|
||||
{
|
||||
PathCounter counter = new PathCounter(metricName, String.format("%s-%02d", year, monthIndex));
|
||||
counter.setCounter(NumberUtils.createLong(value));
|
||||
result.put(counter);
|
||||
}
|
||||
|
||||
monthIndex += 1;
|
||||
}
|
||||
}
|
||||
else if (StringUtils.equals(leaf, "weeks"))
|
||||
{
|
||||
String year = property.getMetricYear();
|
||||
|
||||
StringList values = StatoolInfosUtils.splitWeekValues(property.getValue());
|
||||
int weekIndex = 1;
|
||||
for (String value : values)
|
||||
{
|
||||
if (!StringUtils.isBlank(value))
|
||||
{
|
||||
PathCounter counter = new PathCounter(metricName, String.format("%s-W%02d", year, weekIndex));
|
||||
counter.setCounter(NumberUtils.createLong(value));
|
||||
result.put(counter);
|
||||
}
|
||||
|
||||
weekIndex += 1;
|
||||
}
|
||||
}
|
||||
else if (StringUtils.equals(leaf, "days"))
|
||||
{
|
||||
String year = property.getMetricYear();
|
||||
|
||||
StringList values = StatoolInfosUtils.splitDayValues(property.getValue());
|
||||
LocalDate day = LocalDate.of(Integer.parseInt(year), 1, 1);
|
||||
for (String value : values)
|
||||
{
|
||||
if (!StringUtils.isBlank(value))
|
||||
{
|
||||
String date = day.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
|
||||
PathCounter counter = new PathCounter(metricName, date);
|
||||
counter.setCounter(NumberUtils.createLong(value));
|
||||
result.put(counter);
|
||||
}
|
||||
|
||||
day = day.plusDays(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
probe(configuration);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -352,19 +176,15 @@ public class Prober
|
|||
|
||||
for (String year : years)
|
||||
{
|
||||
// Year stat is complicated to build because needs all the
|
||||
// log of one year.
|
||||
// {
|
||||
// // Year.
|
||||
// PathCounter yearCounter = prefixCounters.get(prefix, year);
|
||||
// if (yearCounter != null)
|
||||
// {
|
||||
// String line = String.format("%s.%s=%s",
|
||||
// yearCounter.getPath(), yearCounter.getTimeMark(),
|
||||
// yearCounter.getCounter());
|
||||
// metrics.appendln(line);
|
||||
// }
|
||||
// }
|
||||
{
|
||||
// Year.
|
||||
PathCounter yearCounter = prefixCounters.get(prefix, year);
|
||||
if (yearCounter != null)
|
||||
{
|
||||
String line = String.format("%s.%s=%s", yearCounter.getPath(), yearCounter.getTimeMark(), yearCounter.getCounter());
|
||||
metrics.appendln(line);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// Months.
|
||||
|
|
|
|||
|
|
@ -26,13 +26,13 @@ import java.util.Locale;
|
|||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||
import fr.devinsy.statoolinfos.util.LineIterator;
|
||||
|
||||
/**
|
||||
|
|
@ -44,12 +44,6 @@ public class HttpAccessLogAnalyzer
|
|||
|
||||
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
|
||||
|
||||
// log_format combined '$remote_addr - $remote_user [$time_local] '
|
||||
// '"$request" $status $body_bytes_sent '
|
||||
// '"$http_referer" "$http_user_agent"';
|
||||
public static final Pattern COMBINED_PATTERN = Pattern.compile(
|
||||
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
|
||||
|
||||
private int errorCount;
|
||||
private PathCounters counters;
|
||||
private VisitCounters visits;
|
||||
|
|
@ -130,22 +124,9 @@ public class HttpAccessLogAnalyzer
|
|||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public void probe(final File file, final String patternRegex) throws IOException
|
||||
public void probe(final File file) throws IOException
|
||||
{
|
||||
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
|
||||
|
||||
//
|
||||
Pattern pattern;
|
||||
if (patternRegex == null)
|
||||
{
|
||||
pattern = COMBINED_PATTERN;
|
||||
}
|
||||
else
|
||||
{
|
||||
pattern = Pattern.compile(patternRegex);
|
||||
}
|
||||
|
||||
//
|
||||
LineIterator iterator = new LineIterator(file);
|
||||
while (iterator.hasNext())
|
||||
{
|
||||
|
|
@ -153,15 +134,7 @@ public class HttpAccessLogAnalyzer
|
|||
|
||||
try
|
||||
{
|
||||
HttpAccessLog log = parseLog(line, pattern);
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
else
|
||||
{
|
||||
probeLog(log);
|
||||
}
|
||||
probeLine(line);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
|
|
@ -178,10 +151,16 @@ public class HttpAccessLogAnalyzer
|
|||
* @param line
|
||||
* the line
|
||||
*/
|
||||
public void probeLog(final HttpAccessLog log)
|
||||
public void probeLine(final String line)
|
||||
{
|
||||
HttpAccessLog log = parseNginxCombinedLog(line);
|
||||
|
||||
// logger.info("==================");
|
||||
if (log != null)
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
else
|
||||
{
|
||||
// logger.info("LINE IS MATCHING [{}]", log);
|
||||
// logger.info(log.getHttpUserAgent().toString());
|
||||
|
|
@ -405,6 +384,30 @@ public class HttpAccessLogAnalyzer
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the http log
|
||||
*/
|
||||
public static HttpAccessLog parseNginxCombinedLog(final String line)
|
||||
{
|
||||
HttpAccessLog result;
|
||||
|
||||
// log_format combined '$remote_addr - $remote_user [$time_local] '
|
||||
// '"$request" $status $body_bytes_sent '
|
||||
// '"$http_referer" "$http_user_agent"';
|
||||
String combinedPattern = "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$";
|
||||
|
||||
Pattern pattern = Pattern.compile(combinedPattern);
|
||||
|
||||
result = parseLog(line, pattern);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
|
|
@ -415,15 +418,30 @@ public class HttpAccessLogAnalyzer
|
|||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public static PathCounters probe(final String source, final String patternRegex) throws IOException, StatoolInfosException
|
||||
public static PathCounters probe(final String source) throws IOException, StatoolInfosException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
HttpAccessLogAnalyzer analyzer = new HttpAccessLogAnalyzer();
|
||||
|
||||
for (File file : FilesUtils.searchByWildcard(source))
|
||||
if (!StringUtils.isBlank(source))
|
||||
{
|
||||
analyzer.probe(file, patternRegex);
|
||||
if (source.endsWith("*"))
|
||||
{
|
||||
String prefix = source.substring(0, source.length() - 1);
|
||||
for (File file : new File(prefix).getParentFile().listFiles())
|
||||
{
|
||||
if (file.getName().startsWith(FilenameUtils.getName(prefix)))
|
||||
{
|
||||
analyzer.probe(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
File file = new File(source);
|
||||
analyzer.probe(file);
|
||||
}
|
||||
}
|
||||
|
||||
result = analyzer.getCounters();
|
||||
|
|
|
|||
|
|
@ -26,12 +26,13 @@ import java.util.Locale;
|
|||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.util.FilesUtils;
|
||||
import fr.devinsy.statoolinfos.util.LineIterator;
|
||||
|
||||
/**
|
||||
|
|
@ -43,9 +44,6 @@ public class HttpErrorLogAnalyzer
|
|||
|
||||
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
|
||||
|
||||
public static final Pattern NGINX_ERROR_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
|
||||
public static final Pattern APACHE_ERROR_PATTERN = Pattern.compile("^\\[(?<time>[^\\]]+)\\]\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
|
||||
|
||||
private PathCounters counters;
|
||||
private int errorCount;
|
||||
|
||||
|
|
@ -84,20 +82,6 @@ public class HttpErrorLogAnalyzer
|
|||
*/
|
||||
public void probe(final File file) throws IOException
|
||||
{
|
||||
if ((file != null) && (!file.isFile()) || (file.exists()))
|
||||
{
|
||||
//
|
||||
Pattern pattern;
|
||||
if (LineIterator.readFirstLine(file).startsWith("["))
|
||||
{
|
||||
pattern = APACHE_ERROR_PATTERN;
|
||||
}
|
||||
else
|
||||
{
|
||||
pattern = NGINX_ERROR_PATTERN;
|
||||
}
|
||||
|
||||
//
|
||||
LineIterator iterator = new LineIterator(file);
|
||||
while (iterator.hasNext())
|
||||
{
|
||||
|
|
@ -105,16 +89,7 @@ public class HttpErrorLogAnalyzer
|
|||
|
||||
try
|
||||
{
|
||||
HttpErrorLog log = parseLog(line, pattern);
|
||||
// logger.info("==================");
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
else
|
||||
{
|
||||
probeLog(log);
|
||||
}
|
||||
probeLine(line);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
|
|
@ -124,7 +99,6 @@ public class HttpErrorLogAnalyzer
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe line.
|
||||
|
|
@ -132,8 +106,20 @@ public class HttpErrorLogAnalyzer
|
|||
* @param line
|
||||
* the line
|
||||
*/
|
||||
public void probeLog(final HttpErrorLog log)
|
||||
public void probeLine(final String line)
|
||||
{
|
||||
HttpErrorLog log = parseNginxLog(line);
|
||||
|
||||
// logger.info("==================");
|
||||
if (log == null)
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
else
|
||||
{
|
||||
// logger.info("LINE IS MATCHING [{}]", log);
|
||||
// logger.info(log.getHttpUserAgent().toString());
|
||||
|
||||
// General HTTP access logs.
|
||||
String year = log.getYear();
|
||||
String yearMonth = log.getYearMonth();
|
||||
|
|
@ -142,6 +128,7 @@ public class HttpErrorLogAnalyzer
|
|||
|
||||
// metrics.http.hits
|
||||
this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
|
||||
// TODO metrics.http.errors.php
|
||||
}
|
||||
|
|
@ -169,7 +156,7 @@ public class HttpErrorLogAnalyzer
|
|||
if (matcher.matches())
|
||||
{
|
||||
result = new HttpErrorLog();
|
||||
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss.SSSSSS yyyy").withLocale(Locale.ENGLISH)));
|
||||
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)));
|
||||
result.setLevel(matcher.group("level"));
|
||||
}
|
||||
else
|
||||
|
|
@ -225,10 +212,25 @@ public class HttpErrorLogAnalyzer
|
|||
|
||||
HttpErrorLogAnalyzer analyzer = new HttpErrorLogAnalyzer();
|
||||
|
||||
for (File file : FilesUtils.searchByWildcard(source))
|
||||
if (!StringUtils.isBlank(source))
|
||||
{
|
||||
if (source.endsWith("*"))
|
||||
{
|
||||
String prefix = source.substring(0, source.length() - 1);
|
||||
for (File file : new File(prefix).getParentFile().listFiles())
|
||||
{
|
||||
if (file.getName().startsWith(FilenameUtils.getName(prefix)))
|
||||
{
|
||||
analyzer.probe(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
File file = new File(source);
|
||||
analyzer.probe(file);
|
||||
}
|
||||
}
|
||||
|
||||
result = analyzer.getCounters();
|
||||
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ public class HttpLogIterator
|
|||
try
|
||||
{
|
||||
String line = this.iterator.next();
|
||||
result = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
|
||||
result = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -18,9 +18,6 @@
|
|||
*/
|
||||
package fr.devinsy.statoolinfos.properties;
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
|
|
@ -28,10 +25,6 @@ import org.apache.commons.lang3.StringUtils;
|
|||
*/
|
||||
public class PathProperty
|
||||
{
|
||||
public static final Pattern METRIC_LABEL_PATTERN = Pattern.compile("^metrics\\.(?<label>\\S+)\\.(name|description|\\d{4}(\\.(months|weeks|days))?)$");
|
||||
public static final Pattern METRIC_NAME_PATTERN = Pattern.compile("^(?<name>metrics\\.\\S+)\\.(name|description|\\d{4}(\\.(months|weeks|days))?)$");
|
||||
public static final Pattern METRIC_YEAR_PATTERN = Pattern.compile("^metrics\\.\\S+\\.(?<year>\\d{4})(\\.(months|weeks|days))?$");
|
||||
|
||||
private String path;
|
||||
private String value;
|
||||
|
||||
|
|
@ -71,101 +64,6 @@ public class PathProperty
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the metric name.
|
||||
*
|
||||
* <pre>
|
||||
* ("service.name") = null
|
||||
* ("metrics.https.visitors.name") = "https.visitors"
|
||||
* ("metrics.https.visitors.2020") = "https.visitors"
|
||||
* ("metrics.https.visitors.2020.months") = "https.visitors"
|
||||
* </pre>
|
||||
*
|
||||
* @return the metric name
|
||||
*/
|
||||
public String getMetricLabel()
|
||||
{
|
||||
String result;
|
||||
|
||||
Matcher matcher = METRIC_LABEL_PATTERN.matcher(this.path);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = matcher.group("label");
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the metric name.
|
||||
*
|
||||
* <pre>
|
||||
* ("service.name") = null
|
||||
* ("metrics.https.visitors.name") = "metrics.https.visitors"
|
||||
* ("metrics.https.visitors.2020") = "metrics.https.visitors"
|
||||
* ("metrics.https.visitors.2020.months") = "metrics.https.visitors"
|
||||
* </pre>
|
||||
*
|
||||
* @return the metric name
|
||||
*/
|
||||
public String getMetricName()
|
||||
{
|
||||
String result;
|
||||
|
||||
Matcher matcher = METRIC_NAME_PATTERN.matcher(this.path);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = matcher.group("name");
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the metric year.
|
||||
*
|
||||
* <pre>
|
||||
* ("service.name") = null
|
||||
* ("metrics.https.visitors.name") = null
|
||||
* ("metrics.https.visitors.2020") = "2020"
|
||||
* ("metrics.https.visitors.2020.months") = "2020"
|
||||
* </pre>
|
||||
*
|
||||
* @return the metric name
|
||||
*/
|
||||
public String getMetricYear()
|
||||
{
|
||||
String result;
|
||||
|
||||
Matcher matcher = METRIC_YEAR_PATTERN.matcher(this.path);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = matcher.group("year");
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the path.
|
||||
*
|
||||
* @return the path
|
||||
*/
|
||||
public String getPath()
|
||||
{
|
||||
return this.path;
|
||||
|
|
@ -176,19 +74,11 @@ public class PathProperty
|
|||
*
|
||||
* @return the prefix
|
||||
*/
|
||||
public String getPrefix()
|
||||
private String getPrefix()
|
||||
{
|
||||
String result;
|
||||
|
||||
int index = this.path.indexOf(".");
|
||||
if (index == -1)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = this.path.substring(0, index);
|
||||
}
|
||||
result = this.path.substring(0, this.path.indexOf("."));
|
||||
|
||||
//
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ package fr.devinsy.statoolinfos.util;
|
|||
import java.io.File;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
|
|
@ -105,44 +104,6 @@ public class FilesUtils
|
|||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search by wildcard.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @return the files
|
||||
*/
|
||||
public static Files searchByWildcard(final String source)
|
||||
{
|
||||
Files result;
|
||||
|
||||
result = new Files();
|
||||
|
||||
if (!StringUtils.isBlank(source))
|
||||
{
|
||||
if (source.endsWith("*"))
|
||||
{
|
||||
String prefix = source.substring(0, source.length() - 1);
|
||||
String shortPrefix = FilenameUtils.getName(prefix);
|
||||
for (File file : new File(prefix).getParentFile().listFiles())
|
||||
{
|
||||
if (file.getName().startsWith(shortPrefix))
|
||||
{
|
||||
result.add(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
File file = new File(source);
|
||||
result.add(file);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* List recursively.
|
||||
*
|
||||
|
|
@ -170,32 +131,4 @@ public class FilesUtils
|
|||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search starting with.
|
||||
*
|
||||
* @param source
|
||||
* the source
|
||||
* @param extensions
|
||||
* the extensions
|
||||
* @return the files
|
||||
*/
|
||||
public static Files searchStartingWith(final File source, final String... extensions)
|
||||
{
|
||||
Files result;
|
||||
|
||||
result = new Files();
|
||||
|
||||
Files full = listRecursively(source);
|
||||
for (File file : full)
|
||||
{
|
||||
if (StringUtils.startsWithAny(file.getName(), extensions))
|
||||
{
|
||||
result.add(file);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -131,42 +131,4 @@ public class LineIterator
|
|||
this.ready = true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read first line.
|
||||
*
|
||||
* @param file
|
||||
* the file
|
||||
* @return the string
|
||||
* @throws IOException
|
||||
* Signals that an I/O exception has occurred.
|
||||
*/
|
||||
public static String readFirstLine(final File file) throws IOException
|
||||
{
|
||||
String result;
|
||||
|
||||
LineIterator iterator = null;
|
||||
try
|
||||
{
|
||||
iterator = new LineIterator(file);
|
||||
if (iterator.hasNext())
|
||||
{
|
||||
result = iterator.next();
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (iterator != null)
|
||||
{
|
||||
iterator.close();
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue