statool/statoolinfosweb
statool/statoolinfosweb
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Activity

Improved LibreQR probing.

Browse source
This commit is contained in:
Christian P. MOMON 2022-02-11 04:21:45 +01:00
parent 067bdfe5bb
commit 676669260b
7 changed files with 361 additions and 279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
src/fr/devinsy/statoolinfos/metrics/PathCounters.java
View file

@ -210,6 +210,27 @@ public class PathCounters extends HashMap<String, PathCounter>
return result;
}
public StringList getNowTimeMarks()
{
StringList result;
result = new StringList();
LocalDate now = LocalDate.now();
String year = TimeMark.yearOf(now).toString();
String yearMonth = TimeMark.yearMonthOf(now).toString();
String yearWeek = TimeMark.yearWeekOf(now).toString();
String date = TimeMark.dayOf(now).toString();
result.add(year);
result.add(yearMonth);
result.add(yearWeek);
result.add(date);
//
return result;
}
/**
* Gets the prefixes.
*
@ -483,6 +504,19 @@ public class PathCounters extends HashMap<String, PathCounter>
return result;
}
/**
* Sets the.
*
* @param value
* the value
* @param path
* the path
*/
public void set(final long value, final String path)
{
set(value, path, getNowTimeMarks());
}
/**
* Sets the.
*
@ -522,4 +556,22 @@ public class PathCounters extends HashMap<String, PathCounter>
counter.setCounter(value);
}
/**
* Sets the.
*
* @param value
* the value
* @param path
* the path
* @param timeMarks
* the time marks
*/
public void set(final long value, final String path, final StringList timeMarks)
{
for (String timeMark : timeMarks)
{
set(value, path, timeMark);
}
}
}

37
src/fr/devinsy/statoolinfos/metrics/Prober.java
View file

@ -38,6 +38,7 @@ import fr.devinsy.statoolinfos.core.DatabaseConfig;
import fr.devinsy.statoolinfos.core.Factory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.statoolinfos.metrics.gitea.GiteaProber;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber;
@ -110,7 +111,7 @@ public class Prober
//
if (types.containsAnyIgnoreCase("HttpAccessLog"))
{
logger.info("== Processing HttpAccessLog.");
logger.info("== Probing HttpAccessLog.");
String source = configuration.getProbeHttpAccessLogSource();
String patternRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("source=[{}]", source);
@ -123,7 +124,7 @@ public class Prober
//
if (types.containsAnyIgnoreCase("HttpErrorLog"))
{
logger.info("== Processing HttpErrorLog.");
logger.info("== Probing HttpErrorLog.");
String source = configuration.getProbeHttpErrorLogSource();
logger.info("source=[{}]", source);
@ -134,18 +135,36 @@ public class Prober
//
if (types.containsAnyIgnoreCase("Framadate"))
{
logger.info("== Processing Framadate.");
logger.info("== Probing Framadate.");
String source = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", source);
// PathCounters data = HttpErrorLogAnalyzer.probe(source);
// PathCounters datafilesPath =
// HttpErrorLogAnalyzer.probe(source);
// counters.putAll(data);
}
//
if (types.containsAnyIgnoreCase("Gitea"))
{
logger.info("== Probing Gitea.");
String httpLogs = configuration.getProbeHttpAccessLogSource();
logger.info("httpLogs=[{}]", httpLogs);
File dataDirectory = configuration.getAsFile("conf.probe.gitea.data");
logger.info("dataPath=[{}]", dataDirectory);
String apiURL = configuration.get("conf.probe.gitea.api.url");
logger.info("apiURL=[{}]", apiURL);
String apiToken = configuration.get("conf.probe.gitea.api.token");
logger.info("apiToken=[{}]", apiURL);
PathCounters metrics = GiteaProber.probe(httpLogs, null, apiURL, apiToken, dataDirectory);
counters.putAll(metrics);
}
//
if (types.containsAnyIgnoreCase("LibreQR"))
{
logger.info("== Processing LibreQR.");
logger.info("== Probing LibreQR.");
String source = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", source);
@ -153,7 +172,7 @@ public class Prober
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("pattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.libreqr.data");
File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles");
logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = LibreQRProber.probe(source, httpAccessLogRegex, dataDirectory);
@ -163,7 +182,7 @@ public class Prober
//
if (types.containsAnyIgnoreCase("Minetest"))
{
logger.info("== Processing Minetest.");
logger.info("== Probing Minetest.");
String source = configuration.get("conf.probe.minetest.logs");
DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.minetest.players");
@ -174,7 +193,7 @@ public class Prober
//
if (types.containsAnyIgnoreCase("Mumble"))
{
logger.info("== Processing Mumble.");
logger.info("== Probing Mumble.");
String source = configuration.get("conf.probe.mumble.logs");
logger.info("source=[{}]", source);
@ -185,7 +204,7 @@ public class Prober
//
if (types.containsAnyIgnoreCase("PrivateBin"))
{
logger.info("== Processing Privatebin.");
logger.info("== Probing Privatebin.");
String httpAccessLogs = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", httpAccessLogs);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();

86
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.http;
import java.io.IOException;
import java.util.Iterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class HttpAccessLogs.
*/
public class HttpAccessLogs implements Iterable<HttpAccessLog>
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogs.class);
private Files source;
private String regex;
/**
* Instantiates a new http access logs.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogs(final Files source) throws IOException
{
this(source, null);
}
/**
* Instantiates a new http log iterator.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogs(final Files source, final String regex) throws IOException
{
this.source = source;
this.regex = regex;
}
/* (non-Javadoc)
* @see java.lang.Iterable#iterator()
*/
@Override
public Iterator<HttpAccessLog> iterator()
{
HttpAccessLogIterator result;
try
{
result = new HttpAccessLogIterator(this.source, this.regex);
}
catch (IOException exception)
{
throw new IllegalArgumentException("Error with iterator.", exception);
}
//
return result;
}
}

95
src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRDataAnalyzer.java
View file

@ -1,95 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.libreqr;
import java.io.File;
import java.io.IOException;
import java.time.LocalDate;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.TimeMark;
import fr.devinsy.statoolinfos.util.FilesUtils;
// TODO: Auto-generated Javadoc
/**
* The Class LibreQRDataAnalyzer.
*/
public class LibreQRDataAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(LibreQRDataAnalyzer.class);
/**
* Instantiates a new http access log prober.
*/
private LibreQRDataAnalyzer()
{
}
/**
* Probe.
*
* @param dataDirectory
* the data directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
{
PathCounters result;
System.out.println("Probing directory [" + dataDirectory + "]");
result = new PathCounters();
if (dataDirectory != null)
{
if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
{
LocalDate now = LocalDate.now();
String year = TimeMark.yearOf(now).toString();
String yearMonth = TimeMark.yearMonthOf(now).toString();
String yearWeek = TimeMark.yearWeekOf(now).toString();
String date = TimeMark.dayOf(now).toString();
// metrics.service.files.bytes
long size = FileUtils.sizeOfDirectory(dataDirectory);
result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
// metrics.pastebins.count
long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date);
}
else
{
System.out.println("WARNING: Privatebin data path is not valid.");
}
}
//
return result;
}
}

150
src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRHttpLogAnalyzer.java
View file

@ -1,150 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.libreqr;
import java.io.IOException;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class LibreQRHttpLogAnalyzer.
*/
public class LibreQRHttpLogAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(LibreQRHttpLogAnalyzer.class);
public static final Pattern USE_PATTERN = Pattern.compile("GET /temp/\\w+\\.png.*");
public static final Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
private PathCounters counters;
private UserCounters users;
private UserCounters ipv4Users;
private UserCounters ipv6Users;
/**
* Instantiates a new http access log prober.
*/
public LibreQRHttpLogAnalyzer()
{
this.counters = new PathCounters();
this.users = new UserCounters();
this.ipv4Users = new UserCounters();
this.ipv6Users = new UserCounters();
}
/**
* Gets the counters.
*
* @return the counters
*/
public PathCounters getCounters()
{
PathCounters result;
result = new PathCounters();
result.putAll(this.counters);
result.putAll(this.users.getCounters("metrics.service.users"));
result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
/**
* Probe log.
*
* @param log
* the log
*/
public void probeLog(final HttpAccessLog log)
{
if (log != null)
{
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
this.users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.barcodes.count
if ((log.getStatus().getCode() == 200) && (CREATE_PATTERN.matcher(log.getRequest()).matches()))
{
this.counters.inc("metrics.barcodes.count", year, yearMonth, yearWeek, date);
}
}
}
/**
* Probe.
*
* @param httpAccessLogs
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
LibreQRHttpLogAnalyzer analyzer = new LibreQRHttpLogAnalyzer();
HttpAccessLogIterator logs = new HttpAccessLogIterator(httpAccessLogFiles, httpRegex);
while (logs.hasNext())
{
analyzer.probeLog(logs.next());
}
result = analyzer.getCounters();
//
return result;
}
}

113
src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java
View file

@ -20,12 +20,18 @@ package fr.devinsy.statoolinfos.metrics.libreqr;
import java.io.File;
import java.io.IOException;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
@ -38,7 +44,7 @@ public class LibreQRProber
/**
* Instantiates a new privatebin prober.
*/
public LibreQRProber()
private LibreQRProber()
{
}
@ -65,32 +71,89 @@ public class LibreQRProber
// metrics.service.users.ipv4
// metrics.service.users.ipv6
// metrics.barcodes.count
result = LibreQRHttpLogAnalyzer.probe(FilesUtils.searchByWildcard(httpLogs), httpLogRegex);
// metrics.libreqr.barcodes.downloads
result = probeHttpAccessLog(FilesUtils.searchByWildcard(httpLogs), httpLogRegex);
/*
local ipv4UserCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year" | grep -vi bot|grep -v check_http | egrep "^$IPV4_PATTERN " | egrep "GET /\?txt=[^&]+&" | cut -d' ' -f 1 | sort | uniq | wc -l )
echo "Nombre d'ipv4 utilisatrices du service = $ipv4UserCount"
local ipv6UserCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year" | grep -vi bot|gre\p -v check_http | egrep "^$IPV6_PATTERN " | egrep "GET /\?txt=[^&]+&" | cut -d' ' -f 1 | sort | uniq | wc -l )
echo "Nombre d'ipv6 utilisatrices du service = $ipv6UserCount"
echo "Nombre total d'ip utilisatrices du service = $((ipv4UserCount+ipv6UserCount))"
local qrcodeCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year" | grep -vi bot|grep -v check_http | egrep "GET /\?txt=[^&]+&" | wc -l)
echo "Nombre de QRCode générés = $qrcodeCount"
local qrcodeDeliveredCount=$(zgrep -h "$monthEnglish" $LOG_PREFIX-access.log*|grep "$year" | grep -vi bot | grep -v check_http | egrep "GET /temp/[^ ]+\.png " | wc -l )
echo "Nombre QRCode renvoyés = $qrcodeDeliveredCount"
local cacheSize=$(du -sh /var/www/qrcode.chapril.org/temp | cut -f 1)
echo "Taille du cache = $cacheSize"
local cacheFileCount=$(find /var/www/qrcode.chapril.org/temp/ -type f | wc -l)
echo "Nombre de fichiers en cache = $cacheFileCount"
*/
// metrics.service.files.bytes, metrics.libreqr.cache.bytes
// metrics.service.files.count, metrics.libreqr.cache.count
result.putAll(DatafilesProber.probe(dataPath));
// metrics.service.files.bytes
result.putAll(LibreQRDataAnalyzer.probe(dataPath));
//
return result;
}
/**
* Probe.
*
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
result = new PathCounters();
//
UserCounters users = new UserCounters();
UserCounters ipv4Users = new UserCounters();
UserCounters ipv6Users = new UserCounters();
Pattern USE_PATTERN = Pattern.compile("GET /temp/\\w+\\.png.*");
Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
//
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
{
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.barcodes.count
if ((log.getStatus().getCode() == 200) && (CREATE_PATTERN.matcher(log.getRequest()).matches()))
{
result.inc("metrics.barcodes.count", year, yearMonth, yearWeek, date);
}
// metrics.libreqr.barcodes.downloads
if (USE_PATTERN.matcher(log.getRequest()).matches())
{
result.inc("metrics.libreqr.barcodes.downloads", year, yearMonth, yearWeek, date);
}
}
//
result.putAll(users.getCounters("metrics.service.users"));
result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;

107
src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java Normal file
View file

@ -0,0 +1,107 @@
/*
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.util;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.strings.StringList;
/**
* The Class DatafilesProber.
*/
public class DatafilesProber
{
private static Logger logger = LoggerFactory.getLogger(DatafilesProber.class);
/**
* Instantiates a new datafiles prober.
*/
private DatafilesProber()
{
}
/**
* Probe.
*
* @param directory
* the directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File directory) throws IOException
{
PathCounters result;
result = probe(directory, "metrics.files");
//
return result;
}
/**
* Probe.
*
* @param directory
* the data directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File directory, final String prefix) throws IOException
{
PathCounters result;
System.out.println("Probing directory [" + directory + "]");
result = new PathCounters();
if ((directory != null) && (directory.exists()) && (directory.isDirectory()))
{
StringList timemarks = result.getNowTimeMarks();
// metrics.service.datafiles.bytes
long size = FileUtils.sizeOfDirectory(directory);
result.set(size, prefix + ".files.bytes", timemarks);
// metrics.service.datafiles.count
long count = FilesUtils.listRecursively(directory.getAbsoluteFile()).size();
result.set(count, prefix + ".files.count", timemarks);
}
else
{
System.out.println("WARNING: datafile path not valid.");
}
//
return result;
}
}