statool/statoolinfosweb
statool/statoolinfosweb
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Activity

Added LibreQR probing. Improved wildcard management.

Browse source
This commit is contained in:
Christian P. MOMON 2022-01-30 03:16:53 +01:00
parent d08eb7a9d7
commit 553068713a
16 changed files with 1025 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -197,6 +197,30 @@ public final class StatoolInfosCLI
return result;
}
/**
* Normalize bash parameter.
*
* @param value
* the value
* @return the string
*/
private static String normalizeBashParameter(final String value)
{
String result;
if (value == null)
{
result = null;
}
else
{
result = StringUtils.trim(value).replace("\\*", "*").replace("\\?", "?");
}
//
return result;
}
/**
* Parses the log filter option.
*
@ -508,6 +532,23 @@ public final class StatoolInfosCLI
}
System.out.println(chrono.format());
}
else if (isMatching(args, "testHttpAccessLog", "\\s*\\S+\\s*"))
{
System.out.println("param1=" + args[1]);
String source = normalizeBashParameter(args[1]);
System.out.println("source=" + source);
Files files = FilesUtils.searchByWildcard(source);
System.out.println(files.size());
try
{
StatoolInfos.testHttpAccessLog(files);
}
catch (Exception exception)
{
logger.error("Error: {}", exception.getMessage());
exception.printStackTrace();
}
}
else
{
System.out.println("Bad usage.");

34
src/fr/devinsy/statoolinfos/core/StatoolInfos.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -34,7 +34,8 @@ import fr.devinsy.statoolinfos.crawl.Crawler;
import fr.devinsy.statoolinfos.htmlize.Htmlizer;
import fr.devinsy.statoolinfos.metrics.Prober;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogParser;
import fr.devinsy.statoolinfos.properties.PathProperties;
import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
import fr.devinsy.statoolinfos.stats.ip.IpStat;
@ -46,6 +47,7 @@ import fr.devinsy.statoolinfos.stats.visitor.VisitorStator;
import fr.devinsy.statoolinfos.uptime.UptimeJournal;
import fr.devinsy.statoolinfos.uptime.UptimeSurveyor;
import fr.devinsy.statoolinfos.util.Chrono;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.LineIterator;
import fr.devinsy.strings.StringList;
import fr.devinsy.strings.StringsUtils;
@ -180,7 +182,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
if (filter.matches(log))
{
@ -230,7 +232,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
if (filter.matches(log))
{
@ -281,7 +283,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
if (filter.matches(log))
{
@ -347,7 +349,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
if (filter.matches(log))
{
stator.putLog(log);
@ -398,7 +400,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
if (filter.matches(log))
{
stator.putLog(log);
@ -449,7 +451,7 @@ public class StatoolInfos
while (iterator.hasNext())
{
String line = iterator.next();
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
if (filter.matches(log))
{
stator.putLog(log);
@ -531,6 +533,22 @@ public class StatoolInfos
}
}
/**
* Test http access log.
*
* @param source
* the source
* @throws IOException
*/
public static void testHttpAccessLog(final Files source) throws IOException
{
HttpAccessLogIterator logs = new HttpAccessLogIterator(source);
while (logs.hasNext())
{
System.out.println(logs.next().toStringLog());
}
}
/**
* Uptime.
*

27
src/fr/devinsy/statoolinfos/metrics/Prober.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -40,6 +40,7 @@ import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber;
import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber;
import fr.devinsy.statoolinfos.metrics.mumble.MumbleProber;
import fr.devinsy.statoolinfos.metrics.privatebin.PrivatebinProber;
@ -134,13 +135,31 @@ public class Prober
if (types.containsAnyIgnoreCase("Framadate"))
{
logger.info("== Processing Framadate.");
String source = configuration.getProbeHttpErrorLogSource();
String source = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", source);
// PathCounters data = HttpErrorLogAnalyzer.probe(source);
// counters.putAll(data);
}
//
if (types.containsAnyIgnoreCase("LibreQR"))
{
logger.info("== Processing LibreQR.");
String source = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", source);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("pattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.libreqr.data");
logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = LibreQRProber.probe(source, httpAccessLogRegex, dataDirectory);
counters.putAll(data);
}
//
if (types.containsAnyIgnoreCase("Minetest"))
{
@ -168,10 +187,10 @@ public class Prober
{
logger.info("== Processing Privatebin.");
String httpAccessLogs = configuration.getProbeHttpAccessLogSource();
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
logger.info("source=[{}]", httpAccessLogs);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("pattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = PrivatebinProber.probe(httpAccessLogs, httpAccessLogRegex, dataDirectory);

54
src/fr/devinsy/statoolinfos/metrics/TimeMarkUtils.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -21,6 +21,7 @@ package fr.devinsy.statoolinfos.metrics;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.YearMonth;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.regex.Pattern;
@ -218,6 +219,23 @@ public class TimeMarkUtils
return result;
}
/**
* Year month of.
*
* @param date
* the date
* @return the string
*/
public static String yearMonthOf(final ZonedDateTime date)
{
String result;
result = date.format(DateTimeFormatter.ofPattern("yyyy-MM", Locale.FRANCE));
//
return result;
}
/**
* Year of.
*
@ -235,6 +253,23 @@ public class TimeMarkUtils
return result;
}
/**
* Year of.
*
* @param date
* the date
* @return the string
*/
public static String yearOf(final ZonedDateTime date)
{
String result;
result = date.format(DateTimeFormatter.ofPattern("yyyy", Locale.FRANCE));
//
return result;
}
/**
* Year week of.
*
@ -251,4 +286,21 @@ public class TimeMarkUtils
//
return result;
}
/**
* Year week of.
*
* @param date
* the date
* @return the string
*/
public static String yearWeekOf(final ZonedDateTime date)
{
String result;
result = YearWeek.from(date).toString();
//
return result;
}
}

49
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -19,6 +19,8 @@
package fr.devinsy.statoolinfos.metrics.http;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.regex.Pattern;
@ -40,7 +42,7 @@ public class HttpAccessLog
private String ip;
private String remoteUser;
private LocalDateTime time;
private ZonedDateTime time;
private String request;
private HttpStatus status;
private long bodyBytesSent;
@ -134,7 +136,7 @@ public class HttpAccessLog
public LocalDateTime getTime()
{
return this.time;
return this.time.toLocalDateTime();
}
public UserAgent getUserAgent()
@ -317,7 +319,18 @@ public class HttpAccessLog
this.status = status;
}
/**
* Sets the time.
*
* @param time
* the new time
*/
public void setTime(final LocalDateTime time)
{
this.time = ZonedDateTime.of(time, ZoneId.systemDefault());
}
public void setTime(final ZonedDateTime time)
{
this.time = time;
}
@ -353,4 +366,34 @@ public class HttpAccessLog
//
return result;
}
/**
* To string log.
*
* @return the string
*/
public String toStringLog()
{
String result;
// "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+)
// \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+)
// (?<bodyBytesSent>\\d+) \"(?<referer>.*)\"
// \"(?<userAgent>[^\"]*)\".*$");
// result = String.format("%s %s %s \\[%s\\] \"%s\" %d %d \"%s\"
// \"%s\"",
result = String.format("%s - %s [%s] \"%s\" %d %d \"%s\" \"%s\"",
this.ip,
this.remoteUser,
this.time.format(DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH)),
this.request,
this.status.getCode(),
this.bodyBytesSent,
this.referer,
this.userAgent);
//
return result;
}
}

46
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java
View file

@ -20,9 +20,6 @@ package fr.devinsy.statoolinfos.metrics.http;
import java.io.File;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -43,12 +40,6 @@ public class HttpAccessLogAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogAnalyzer.class);
// log_format combined '$remote_addr - $remote_user [$time_local] '
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
public static final Pattern COMBINED_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>.*)\" \"(?<userAgent>[^\"]*)\".*$");
private int errorCount;
private PathCounters counters;
private IpCounters ips;
@ -149,7 +140,7 @@ public class HttpAccessLogAnalyzer
Pattern pattern;
if (patternRegex == null)
{
pattern = COMBINED_PATTERN;
pattern = HttpAccessLogParser.COMBINED_PATTERN;
}
else
{
@ -164,7 +155,7 @@ public class HttpAccessLogAnalyzer
try
{
HttpAccessLog log = parseLog(line, pattern);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
@ -410,39 +401,6 @@ public class HttpAccessLogAnalyzer
return result;
}
/**
* Parses the log.
*
* @param line
* the line
* @return the http log
*/
public static HttpAccessLog parseLog(final String line, final Pattern pattern)
{
HttpAccessLog result;
Matcher matcher = pattern.matcher(line);
if (matcher.matches())
{
result = new HttpAccessLog();
result.setIp(matcher.group("remoteAddress"));
result.setRemoteUser(matcher.group("remoteUser"));
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
result.setRequest(matcher.group("request"));
result.setStatus(HttpStatusTable.instance().get(matcher.group("status")));
result.setBodyBytesSent(Long.valueOf(matcher.group("bodyBytesSent")));
result.setReferer(matcher.group("referer"));
result.setUserAgent(new UserAgent(matcher.group("userAgent")));
}
else
{
result = null;
}
//
return result;
}
/**
* Probe.
*

159
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java Normal file
View file

@ -0,0 +1,159 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.http;
import java.io.IOException;
import java.util.Iterator;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.FilesLineIterator;
/**
* The Class HttpAccessLogIterator.
*/
public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogIterator.class);
private FilesLineIterator lineIterator;
private Pattern pattern;
private HttpAccessLog nextLog;
/**
* Instantiates a new http access log iterator.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogIterator(final Files source) throws IOException
{
this(source, null);
}
/**
* Instantiates a new http log iterator.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogIterator(final Files source, final String regex) throws IOException
{
this.lineIterator = new FilesLineIterator(source);
this.nextLog = null;
if (regex == null)
{
this.pattern = HttpAccessLogParser.COMBINED_PATTERN;
}
else
{
this.pattern = Pattern.compile(regex);
}
}
/* (non-Javadoc)
* @see java.util.Iterator#hasNext()
*/
@Override
public boolean hasNext()
{
boolean result;
preload();
if (this.nextLog == null)
{
result = false;
}
else
{
result = true;
}
//
return result;
}
/* (non-Javadoc)
* @see java.util.Iterator#next()
*/
@Override
public HttpAccessLog next()
{
HttpAccessLog result;
preload();
result = this.nextLog;
this.nextLog = null;
//
return result;
}
/**
* Forward.
*/
private void preload()
{
if (this.nextLog == null)
{
boolean ended = false;
while (!ended)
{
if (this.lineIterator.hasNext())
{
String line = this.lineIterator.next();
try
{
HttpAccessLog log = HttpAccessLogParser.parseLog(line, this.pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
this.nextLog = log;
ended = true;
}
}
catch (Exception exception)
{
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
exception.printStackTrace();
}
}
else
{
this.nextLog = null;
ended = true;
}
}
}
}
}

94
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogParser.java Normal file
View file

@ -0,0 +1,94 @@
/*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.http;
import java.io.File;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator;
/**
* The Class HttpAccessLogParser.
*/
public class HttpAccessLogParser
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogParser.class);
// log_format combined '$remote_addr - $remote_user [$time_local] '
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
public static final Pattern COMBINED_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>.*)\" \"(?<userAgent>[^\"]*)\".*$");
/**
* Instantiates a new http access log parser.
*/
private HttpAccessLogParser()
{
}
/**
* Parses the log.
*
* @param line
* the line
* @return the http log
*/
public static HttpAccessLog parseLog(final String line, final Pattern pattern)
{
HttpAccessLog result;
Matcher matcher = pattern.matcher(line);
if (matcher.matches())
{
ZonedDateTime a = ZonedDateTime.now();
LocalDateTime b = a.toLocalDateTime();
result = new HttpAccessLog();
result.setIp(matcher.group("remoteAddress"));
result.setRemoteUser(matcher.group("remoteUser"));
result.setTime(ZonedDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
result.setRequest(matcher.group("request"));
result.setStatus(HttpStatusTable.instance().get(matcher.group("status")));
result.setBodyBytesSent(Long.valueOf(matcher.group("bodyBytesSent")));
result.setReferer(matcher.group("referer"));
result.setUserAgent(new UserAgent(matcher.group("userAgent")));
}
else
{
result = null;
}
//
return result;
}
}

4
src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -97,7 +97,7 @@ public class HttpLogIterator
try
{
String line = this.iterator.next();
result = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
result = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
}
catch (Exception exception)
{

95
src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRDataAnalyzer.java Normal file
View file

@ -0,0 +1,95 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.libreqr;
import java.io.File;
import java.io.IOException;
import java.time.LocalDate;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.TimeMark;
import fr.devinsy.statoolinfos.util.FilesUtils;
// TODO: Auto-generated Javadoc
/**
* The Class LibreQRDataAnalyzer.
*/
public class LibreQRDataAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(LibreQRDataAnalyzer.class);
/**
* Instantiates a new http access log prober.
*/
private LibreQRDataAnalyzer()
{
}
/**
* Probe.
*
* @param dataDirectory
* the data directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
{
PathCounters result;
System.out.println("Probing directory [" + dataDirectory + "]");
result = new PathCounters();
if (dataDirectory != null)
{
if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
{
LocalDate now = LocalDate.now();
String year = TimeMark.yearOf(now).toString();
String yearMonth = TimeMark.yearMonthOf(now).toString();
String yearWeek = TimeMark.yearWeekOf(now).toString();
String date = TimeMark.dayOf(now).toString();
// metrics.service.files.bytes
long size = FileUtils.sizeOfDirectory(dataDirectory);
result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
// metrics.pastebins.count
long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date);
}
else
{
System.out.println("WARNING: Privatebin data path is not valid.");
}
}
//
return result;
}
}

144
src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRHttpLogAnalyzer.java Normal file
View file

@ -0,0 +1,144 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.libreqr;
import java.io.IOException;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class LibreQRHttpLogAnalyzer.
*/
public class LibreQRHttpLogAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(LibreQRHttpLogAnalyzer.class);
public static final Pattern USE_PATTERN = Pattern.compile("GET /temp/\\w+\\.png.*");
private PathCounters counters;
private UserCounters users;
private UserCounters ipv4Users;
private UserCounters ipv6Users;
/**
* Instantiates a new http access log prober.
*/
public LibreQRHttpLogAnalyzer()
{
this.counters = new PathCounters();
this.users = new UserCounters();
this.ipv4Users = new UserCounters();
this.ipv6Users = new UserCounters();
}
/**
* Gets the counters.
*
* @return the counters
*/
public PathCounters getCounters()
{
PathCounters result;
result = new PathCounters();
result.putAll(this.counters);
result.putAll(this.users.getCounters("metrics.service.users"));
result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
/**
* Probe log.
*
* @param log
* the log
*/
public void probeLog(final HttpAccessLog log)
{
if (log != null)
{
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
// metrics.barcodes.count
if (USE_PATTERN.matcher(log.getRequest()).matches())
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
this.users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
}
}
/**
* Probe.
*
* @param httpAccessLogs
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
LibreQRHttpLogAnalyzer analyzer = new LibreQRHttpLogAnalyzer();
HttpAccessLogIterator logs = new HttpAccessLogIterator(httpAccessLogFiles, httpRegex);
while (logs.hasNext())
{
analyzer.probeLog(logs.next());
}
result = analyzer.getCounters();
//
return result;
}
}

98
src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java Normal file
View file

@ -0,0 +1,98 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.libreqr;
import java.io.File;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
* The Class LibreQRProber.
*/
public class LibreQRProber
{
private static Logger logger = LoggerFactory.getLogger(LibreQRProber.class);
/**
* Instantiates a new privatebin prober.
*/
public LibreQRProber()
{
}
/**
* Probe.
*
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
{
PathCounters result;
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
// metrics.barcodes.count
result = LibreQRHttpLogAnalyzer.probe(FilesUtils.searchByWildcard(httpLogs), httpLogRegex);
/*
local ipv4UserCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year" | grep -vi bot|grep -v check_http | egrep "^$IPV4_PATTERN " | egrep "GET /\?txt=[^&]+&" | cut -d' ' -f 1 | sort | uniq | wc -l )
echo "Nombre d'ipv4 utilisatrices du service = $ipv4UserCount"
local ipv6UserCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year" | grep -vi bot|gre\p -v check_http | egrep "^$IPV6_PATTERN " | egrep "GET /\?txt=[^&]+&" | cut -d' ' -f 1 | sort | uniq | wc -l )
echo "Nombre d'ipv6 utilisatrices du service = $ipv6UserCount"
echo "Nombre total d'ip utilisatrices du service = $((ipv4UserCount+ipv6UserCount))"
local qrcodeCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year" | grep -vi bot|grep -v check_http | egrep "GET /\?txt=[^&]+&" | wc -l)
echo "Nombre de QRCode générés = $qrcodeCount"
local qrcodeDeliveredCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year" | grep -vi bot | grep -v check_http | egrep "GET /temp/[^ ]+\.png " | wc -l )
echo "Nombre QRCode renvoyés = $qrcodeDeliveredCount"
local cacheSize=$(du -sh /var/www/qrcode.chapril.org/temp | cut -f 1)
echo "Taille du cache = $cacheSize"
local cacheFileCount=$(find /var/www/qrcode.chapril.org/temp/ -type f | wc -l)
echo "Nombre de fichiers en cache = $cacheFileCount"
*/
// metrics.service.files.bytes
result.putAll(LibreQRDataAnalyzer.probe(dataPath));
//
return result;
}
}

8
src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -30,7 +30,7 @@ import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogParser;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator;
@ -95,7 +95,7 @@ public class PrivatebinHttpLogAnalyzer
Pattern pattern;
if (patternRegex == null)
{
pattern = HttpAccessLogAnalyzer.COMBINED_PATTERN;
pattern = HttpAccessLogParser.COMBINED_PATTERN;
}
else
{
@ -110,7 +110,7 @@ public class PrivatebinHttpLogAnalyzer
try
{
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, pattern);
HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);

103
src/fr/devinsy/statoolinfos/util/Files.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -49,6 +49,56 @@ public class Files extends ArrayList<File>
super(initialCapacity);
}
/**
* Keep directories.
*
* @return the files
*/
public Files keepDirectoryType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isDirectory())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Keep file type.
*
* @return the files
*/
public Files keepFileType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isFile())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Keep.
*
@ -96,7 +146,6 @@ public class Files extends ArrayList<File>
while (iterator.hasNext())
{
File file = iterator.next();
if (!pattern.matcher(file.getAbsolutePath()).matches())
{
iterator.remove();
@ -109,56 +158,6 @@ public class Files extends ArrayList<File>
return result;
}
/**
* Keep directories.
*
* @return the files
*/
public Files keepDirectoryType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isDirectory())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Keep file type.
*
* @return the files
*/
public Files keepFileType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isFile())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Removes the containing.
*

181
src/fr/devinsy/statoolinfos/util/FilesLineIterator.java Normal file
View file

@ -0,0 +1,181 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.util;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class WildcardFileLineIterator.
*/
public class FilesLineIterator implements Iterator<String>
{
private static Logger logger = LoggerFactory.getLogger(FilesLineIterator.class);
private Iterator<File> fileIterator;
private LineIterator lineIterator;
private boolean print;
/**
* Instantiates a new wildcard file line iterator.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public FilesLineIterator(final Files source)
{
this.fileIterator = source.iterator();
this.lineIterator = null;
setPrintOn();
}
/**
* Forward.
*/
private void forward()
{
boolean ended = false;
while (!ended)
{
try
{
if (this.lineIterator == null)
{
if (this.fileIterator.hasNext())
{
File file = this.fileIterator.next();
if (this.print)
{
System.out.println("Iterating file [" + file.getAbsolutePath() + "]");
}
this.lineIterator = new LineIterator(file);
}
else
{
ended = true;
}
}
else if (this.lineIterator.hasNext())
{
ended = true;
}
else
{
this.lineIterator.close();
if (this.fileIterator.hasNext())
{
File file = this.fileIterator.next();
if (this.print)
{
System.out.println("Iterating file [" + file.getAbsolutePath() + "]");
}
this.lineIterator = new LineIterator(file);
}
else
{
this.lineIterator = null;
ended = true;
}
}
}
catch (IOException exception)
{
logger.error("ERROR READING.", exception);
if (this.lineIterator != null)
{
this.lineIterator.close();
this.lineIterator = null;
}
}
}
}
/* (non-Javadoc)
* @see java.util.Iterator#hasNext()
*/
@Override
public boolean hasNext()
{
boolean result;
forward();
if ((this.lineIterator == null) && (!this.fileIterator.hasNext()))
{
result = false;
}
else
{
result = true;
}
//
return result;
}
/* (non-Javadoc)
* @see java.util.Iterator#next()
*/
@Override
public String next()
{
String result;
try
{
forward();
if (this.lineIterator == null)
{
result = null;
}
else
{
result = this.lineIterator.next();
}
}
catch (IOException exception)
{
throw new IllegalArgumentException("ERROR READING.", exception);
}
//
return result;
}
/**
* Sets the print off.
*/
public void setPrintOff()
{
this.print = false;
}
public void setPrintOn()
{
this.print = true;
}
}

8
src/fr/devinsy/statoolinfos/util/FilesUtils.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -121,7 +121,9 @@ public class FilesUtils
{
// Get the first parent without wildcard.
File parent = null;
File current = new File(source);
File path = new File(source).getAbsoluteFile();
File current = path;
boolean ended = false;
while (!ended)
{
@ -146,7 +148,7 @@ public class FilesUtils
//
System.out.println("parent=" + parent);
String regex = source.replace(".", "\\.").replace("?", ".").replace("*", ".*");
String regex = path.toString().replace(".", "\\.").replace("?", ".").replace("*", ".*");
result = listRecursively(parent).keepPath(regex);
}