Compare commits

..

No commits in common. "d7fad84d113a9d5c5f9668815da2149b423499e0" and "5080c980d8cba653d6d43ed487279906c5625c54" have entirely different histories.

21 changed files with 166 additions and 1043 deletions

View file

@ -13,10 +13,9 @@ if [[ "$javaCheck" =~ ^/.* ]]; then
# Optional system properties:
# LOGFILE="-Dlog4j2.configurationFile=../../log4j2.properties"
# LOGLEVEL="-Dlog4j2.level=ERROR"
# IPV6ONLY="-Djava.net.preferIPv4Stack=false -Djava.net.preferIPv6Addresses=true"
# https://logging.apache.org/log4j/log4j-2.11.2/manual/configuration.html#System_Properties
java -Djava.awt.headless=true $IPV6ONLY $LOGFILE $LOGLEVEL -jar "$(dirname "$0")"/statoolinfos.jar $@
java -Djava.awt.headless=true $LOGFILE $LOGLEVEL -jar "$(dirname "$0")"/statoolinfos.jar $@
else
echo "Java requirement............... MISSING"
fi

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -352,21 +352,6 @@ public class Configuration extends PathPropertyList
return result;
}
/**
* Gets the probe http access log path filter.
*
* @return the probe http access log path filter
*/
public String getProbeHttpAccessLogPathFilter()
{
String result;
result = get("conf.probe.httpaccesslog.pathfilter");
//
return result;
}
/**
* Gets the probe http access log pattern.
*

View file

@ -1,72 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.1"
id="svg2"
xml:space="preserve"
width="85.333336"
height="85.333336"
viewBox="0 0 85.333336 85.333336"
sodipodi:docname="folder-mono.svg"
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"><metadata
id="metadata8"><rdf:RDF><cc:Work
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
id="defs6"><clipPath
clipPathUnits="userSpaceOnUse"
id="clipPath18"><path
d="M 0,64 H 64 V 0 H 0 Z"
id="path16"
inkscape:connector-curvature="0" /></clipPath></defs><sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="640"
inkscape:window-height="480"
id="namedview4"
showgrid="false"
inkscape:zoom="2.7656249"
inkscape:cx="42.666668"
inkscape:cy="42.666668"
inkscape:window-x="811"
inkscape:window-y="140"
inkscape:window-maximized="0"
inkscape:current-layer="g10" /><g
id="g10"
inkscape:groupmode="layer"
inkscape:label="Elegant_circle-icons"
transform="matrix(1.3333333,0,0,-1.3333333,0,85.333333)"><g
id="g12"><g
id="g14"
clip-path="url(#clipPath18)"><g
id="g20"
transform="translate(48.5,48)"><path
d="m 0,0 h -12.5 c -1.38,0 -2.5,-1.103 -2.5,-2.462 0,0 -0.124,-0.777 -0.658,-1.461 H 2.5 v 1.461 C 2.5,-1.103 1.38,0 0,0"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path22"
inkscape:connector-curvature="0" /></g><g
id="g24"
transform="translate(13,40.6543)"><path
d="m 0,0 v -22.192 c 0,-0.806 0.673,-1.462 1.5,-1.462 h 35 c 0.827,0 1.5,0.656 1.5,1.462 V 1.423 H 1.5 C 0.659,1.423 0,0.798 0,0"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path26"
inkscape:connector-curvature="0" /></g><g
id="g28"
transform="translate(51,44.0771)"><path
d="m 0,0 h -18.158 c 0.534,0.684 0.658,1.461 0.658,1.461 0,1.359 1.12,2.462 2.5,2.462 H -2.5 C -1.12,3.923 0,2.82 0,1.461 Z m 0,-25.615 c 0,-0.806 -0.673,-1.462 -1.5,-1.462 h -35 c -0.827,0 -1.5,0.656 -1.5,1.462 v 22.192 c 0,0.798 0.659,1.423 1.5,1.423 L 0,-2 Z m -19,45.538 c -17.673,0 -32,-14.327 -32,-32 0,-17.673 14.327,-32 32,-32 17.673,0 32,14.327 32,32 0,17.673 -14.327,32 -32,32"
style="fill:#4f5d73;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path30"
inkscape:connector-curvature="0" /></g></g></g></g></svg>

Before

Width:  |  Height:  |  Size: 3.3 KiB

View file

@ -1,89 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.1"
id="svg2"
xml:space="preserve"
width="85.333336"
height="85.333336"
viewBox="0 0 85.333336 85.333336"
sodipodi:docname="folder.svg"
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"><metadata
id="metadata8"><rdf:RDF><cc:Work
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
id="defs6"><clipPath
clipPathUnits="userSpaceOnUse"
id="clipPath18"><path
d="M 0,64 H 64 V 0 H 0 Z"
id="path16"
inkscape:connector-curvature="0" /></clipPath><clipPath
clipPathUnits="userSpaceOnUse"
id="clipPath30"><path
d="M 12,41.077 H 52 V 14 H 12 Z"
id="path28"
inkscape:connector-curvature="0" /></clipPath></defs><sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="640"
inkscape:window-height="480"
id="namedview4"
showgrid="false"
inkscape:zoom="2.7656249"
inkscape:cx="42.666668"
inkscape:cy="42.666668"
inkscape:window-x="905"
inkscape:window-y="265"
inkscape:window-maximized="0"
inkscape:current-layer="g10" /><g
id="g10"
inkscape:groupmode="layer"
inkscape:label="Elegant_circle-icons"
transform="matrix(1.3333333,0,0,-1.3333333,0,85.333333)"><g
id="g12"><g
id="g14"
clip-path="url(#clipPath18)"><g
id="g20"
transform="translate(64,32)"><path
d="m 0,0 c 0,-17.673 -14.327,-32 -32,-32 -17.673,0 -32,14.327 -32,32 0,17.673 14.327,32 32,32 C -14.327,32 0,17.673 0,0"
style="fill:#e0995e;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path22"
inkscape:connector-curvature="0" /></g><g
id="g24"><g
id="g26" /><g
id="g38"><g
clip-path="url(#clipPath30)"
id="g36"
style="opacity:0.19999701"><g
transform="translate(32,41.077)"
id="g34"><path
d="M 0,0 H -0.417 -17.5 C -18.88,0 -20,-1.063 -20,-2.422 v -22.193 c 0,-1.359 1.12,-2.462 2.5,-2.462 h 35 c 1.38,0 2.5,1.103 2.5,2.462 V -4.923 -2.422 0 Z"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path32"
inkscape:connector-curvature="0" /></g></g></g></g><g
id="g40"
transform="translate(32,43.077)"><path
d="M 0,0 H -0.417 -17.5 C -18.88,0 -20,-1.063 -20,-2.422 v -22.193 c 0,-1.359 1.12,-2.462 2.5,-2.462 h 35 c 1.38,0 2.5,1.103 2.5,2.462 V -4.923 -2.422 0 Z"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path42"
inkscape:connector-curvature="0" /></g><g
id="g44"
transform="translate(52,45.5385)"><path
d="M 0,0 C 0,1.359 -1.12,2.462 -2.5,2.462 H -15 c -1.38,0 -2.5,-1.103 -2.5,-2.462 0,0 -0.325,-2.257 -2.5,-2.461 H 0 Z"
style="fill:#e0e0d1;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path46"
inkscape:connector-curvature="0" /></g></g></g></g></svg>

Before

Width:  |  Height:  |  Size: 3.8 KiB

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -42,7 +42,6 @@ import fr.devinsy.statoolinfos.metrics.etherpad.EtherpadProber;
import fr.devinsy.statoolinfos.metrics.gitea.GiteaProber;
import fr.devinsy.statoolinfos.metrics.gsl.GSLProber;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber;
import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber;
@ -117,15 +116,11 @@ public class Prober
{
logger.info("== Probing HttpAccessLog.");
String source = configuration.getProbeHttpAccessLogSource();
String pattern = configuration.getProbeHttpAccessLogPattern();
String pathFilter = configuration.getProbeHttpAccessLogPathFilter();
String patternRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("source=[{}]", source);
logger.info("pattern=[{}]", pattern);
logger.info("path=[{}]", pathFilter);
logger.info("pattern=[{}]", patternRegex);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, pathFilter);
PathCounters data = HttpAccessLogAnalyzer.probe(httpAccessLogs);
PathCounters data = HttpAccessLogAnalyzer.probe(source, patternRegex);
counters.putAll(data);
}
@ -144,20 +139,16 @@ public class Prober
if (types.containsAnyIgnoreCase("Etherpad"))
{
logger.info("== Probing Etherpad.");
String httpLogs = configuration.getProbeHttpAccessLogSource();
logger.info("httpLogs=[{}]", httpLogs);
String logs = configuration.get("conf.probe.etherpad.logs");
logger.info("logs=[{}]", logs);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("httpAccessPattern=[{}]", httpAccessLogRegex);
DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.etherpad");
logger.info("database={}", database.toString());
String httpLogs = configuration.getProbeHttpAccessLogSource();
String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
String httpAccessPathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("httpLogs=[{}]", httpLogs);
logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
logger.info("httpAccessPathFilter=[{}]", httpAccessPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessPathFilter);
PathCounters metrics = EtherpadProber.probe(httpAccessLogs, FilesUtils.searchByWildcard(logs), database);
PathCounters metrics = EtherpadProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, FilesUtils.searchByWildcard(logs), database);
counters.putAll(metrics);
}
@ -177,6 +168,10 @@ public class Prober
if (types.containsAnyIgnoreCase("Gitea"))
{
logger.info("== Probing Gitea.");
String httpLogs = configuration.getProbeHttpAccessLogSource();
logger.info("httpLogs=[{}]", httpLogs);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("httpAccessPattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.gitea.data");
logger.info("dataPath=[{}]", dataDirectory);
String apiURL = configuration.get("conf.probe.gitea.api.url");
@ -186,15 +181,7 @@ public class Prober
DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.gitea");
logger.info("database={}", database.toString());
String httpLogs = configuration.getProbeHttpAccessLogSource();
String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("httpLogs=[{}]", httpLogs);
logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessLogPathFilter);
PathCounters metrics = GiteaProber.probe(httpAccessLogs, apiURL, apiToken, dataDirectory, database);
PathCounters metrics = GiteaProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, apiURL, apiToken, dataDirectory, database);
counters.putAll(metrics);
}
@ -215,18 +202,16 @@ public class Prober
{
logger.info("== Probing LibreQR.");
String httpAccessSource = configuration.getProbeHttpAccessLogSource();
String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("httpAccessSource=[{}]", httpAccessSource);
logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessSource), httpAccessLogPattern, httpAccessLogPathFilter);
String source = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", source);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("pattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles");
logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = LibreQRProber.probe(httpAccessLogs, dataDirectory);
PathCounters data = LibreQRProber.probe(FilesUtils.searchByWildcard(source), httpAccessLogRegex, dataDirectory);
counters.putAll(data);
}
@ -260,18 +245,14 @@ public class Prober
if (types.containsAnyIgnoreCase("PrivateBin"))
{
logger.info("== Probing Privatebin.");
String httpAccessLogSource = configuration.getProbeHttpAccessLogSource();
String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
logger.info("source=[{}]", httpAccessLogSource);
logger.info("pattern=[{}]", httpAccessLogPattern);
logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessLogSource), httpAccessLogPattern, httpAccessLogPathFilter);
String httpAccessLogs = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", httpAccessLogs);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("pattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = PrivatebinProber.probe(httpAccessLogs, dataDirectory);
PathCounters data = PrivatebinProber.probe(FilesUtils.searchByWildcard(httpAccessLogs), httpAccessLogRegex, dataDirectory);
counters.putAll(data);
}

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -28,7 +28,8 @@ import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class EtherpadHttpLogAnalyzer.
@ -115,23 +116,26 @@ public class EtherpadHttpLogAnalyzer
/**
* Probe.
*
* @param logs
* the logs
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final HttpAccessLogs logs) throws IOException, StatoolInfosException
public static PathCounters probe(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
EtherpadHttpLogAnalyzer analyzer = new EtherpadHttpLogAnalyzer();
for (HttpAccessLog log : logs)
HttpAccessLogIterator logs = new HttpAccessLogIterator(httpAccessLogFiles, httpRegex);
while (logs.hasNext())
{
analyzer.probeLog(log);
analyzer.probeLog(logs.next());
}
result = analyzer.getCounters();

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2022-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.DatabaseConfig;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatabaseProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.sql.SQLDatabase;
@ -50,19 +49,19 @@ public class EtherpadProber
/**
* Probe.
*
* @param httpAccessLogs
* the http access logs
* @param logs
* the logs
* @param databaseConfig
* the database config
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final HttpAccessLogs httpAccessLogs, final Files logs, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final Files logs, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException
{
PathCounters result;
@ -71,7 +70,7 @@ public class EtherpadProber
// metrics.service.users.ipv6
if (logs.isEmpty())
{
result = EtherpadHttpLogAnalyzer.probe(httpAccessLogs);
result = EtherpadHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
}
else
{
@ -146,25 +145,6 @@ public class EtherpadProber
{
System.out.println("Etherpad Database undefined.");
}
// https://etherpad.org/doc/v1.8.18/#index_statistics
// Etherpad keeps track of the goings-on inside the edit machinery.
// If you'd like to have a look at this, just point your browser to
// /stats.
// We currently measure:
//
// totalUsers (counter)
// connects (meter)
// disconnects (meter)
// pendingEdits (counter)
// edits (timer)
// failedChangesets (meter)
// httpRequests (timer)
// http500 (meter)
// memoryUsage (gauge)
//
// https://pad.libre-service.eu/stats/
}
catch (SQLException exception)
{

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2022-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -37,6 +37,7 @@ import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.http.HttpStatusCategory;
import fr.devinsy.statoolinfos.metrics.util.DatabaseProber;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.sql.SQLDatabase;
import fr.devinsy.strings.StringList;
@ -57,23 +58,19 @@ public class GiteaProber
/**
* Probe.
*
* @param httpAccessLogs
* the http access logs
* @param apiURL
* the api URL
* @param apiToken
* the api token
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @param databaseConfig
* the database config
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final HttpAccessLogs httpAccessLogs, final String apiURL, final String apiToken, final File dataPath, final DatabaseConfig databaseConfig)
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final String apiURL, final String apiToken, final File dataPath, final DatabaseConfig databaseConfig)
throws IOException, StatoolInfosException
{
PathCounters result;
@ -81,7 +78,7 @@ public class GiteaProber
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
result = probeHttpAccessLog(httpAccessLogs);
result = probeHttpAccessLog(httpLogs, httpLogRegex);
// metrics.service.database.bytes
try
@ -168,15 +165,17 @@ public class GiteaProber
/**
* Probe http access log.
*
* @param logs
* the logs
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
private static PathCounters probeHttpAccessLog(final HttpAccessLogs logs) throws IOException, StatoolInfosException
private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
@ -296,7 +295,7 @@ public class GiteaProber
Pattern USE_PATTERN = Pattern.compile(regex);
//
for (HttpAccessLog log : logs)
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
{
// General HTTP access logs.
String year = log.getYear();

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -33,7 +33,6 @@ import fr.devinsy.strings.StringList;
/**
* The Class HttpAccessLog.
*
*/
public class HttpAccessLog
{

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -18,6 +18,7 @@
*/
package fr.devinsy.statoolinfos.metrics.http;
import java.io.File;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -29,6 +30,8 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.IpCounters;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator;
/**
* The Class HttpAccessLogProber.
@ -37,7 +40,6 @@ public class HttpAccessLogAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogAnalyzer.class);
private int ignoredLineCount;
private int errorCount;
private PathCounters counters;
private IpCounters ips;
@ -62,9 +64,6 @@ public class HttpAccessLogAnalyzer
*/
public HttpAccessLogAnalyzer()
{
this.errorCount = 0;
this.ignoredLineCount = 0;
this.counters = new PathCounters();
this.ips = new IpCounters();
this.ipv4 = new IpCounters();
@ -123,6 +122,58 @@ public class HttpAccessLogAnalyzer
return result;
}
/**
* Probe.
*
* @param file
* the file
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public void probe(final File file, final String patternRegex) throws IOException
{
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
//
Pattern pattern;
if (StringUtils.isBlank(patternRegex))
{
pattern = HttpAccessLogParser.COMBINED_PATTERN;
}
else
{
pattern = Pattern.compile(patternRegex);
}
//
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
try
{
HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
probeLog(log);
}
}
catch (Exception exception)
{
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
exception.printStackTrace();
this.errorCount += 1;
}
}
}
/**
* Probe log.
*
@ -353,23 +404,22 @@ public class HttpAccessLogAnalyzer
/**
* Probe.
*
* @param logs
* the logs
* @return the path counters
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final HttpAccessLogs logs) throws IOException, StatoolInfosException
public static PathCounters probe(final String source, final String patternRegex) throws IOException, StatoolInfosException
{
PathCounters result;
HttpAccessLogAnalyzer analyzer = new HttpAccessLogAnalyzer();
for (HttpAccessLog log : logs)
for (File file : FilesUtils.searchByWildcard(source))
{
analyzer.probeLog(log);
analyzer.probe(file, patternRegex);
}
result = analyzer.getCounters();

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2022-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.Iterator;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -38,10 +37,7 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
private FilesLineIterator lineIterator;
private Pattern pattern;
private Pattern requestFilter;
private HttpAccessLog nextLog;
private int errorCount;
private int ignoredLineCount;
/**
* Instantiates a new http access log iterator.
@ -53,22 +49,7 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
*/
public HttpAccessLogIterator(final Files source) throws IOException
{
this(source, null, null);
}
/**
* Instantiates a new http access log iterator.
*
* @param source
* the source
* @param regex
* the regex
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogIterator(final Files source, final String regex) throws IOException
{
this(source, regex, null);
this(source, null);
}
/**
@ -79,14 +60,12 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogIterator(final Files source, final String regex, final String pathFilter) throws IOException
public HttpAccessLogIterator(final Files source, final String regex) throws IOException
{
this.lineIterator = new FilesLineIterator(source);
this.nextLog = null;
this.errorCount = 0;
this.ignoredLineCount = 0;
if (StringUtils.isBlank(regex))
if (regex == null)
{
this.pattern = HttpAccessLogParser.COMBINED_PATTERN;
}
@ -94,37 +73,6 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
{
this.pattern = Pattern.compile(regex);
}
if (StringUtils.isBlank(pathFilter))
{
this.requestFilter = null;
}
else
{
String filter = "^\\S+ " + pathFilter + " .*$";
System.out.println("requestFilter=" + filter);
this.requestFilter = Pattern.compile(filter);
}
}
/**
* Gets the error count.
*
* @return the error count
*/
public int getErrorCount()
{
return this.errorCount;
}
/**
* Gets the ignored line count.
*
* @return the ignored line count
*/
public int getIgnoredLineCount()
{
return this.ignoredLineCount;
}
/* (non-Javadoc)
@ -187,17 +135,11 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
this.errorCount += 1;
}
else if ((this.requestFilter == null) || (this.requestFilter.matcher(log.getRequest()).matches()))
{
this.nextLog = log;
ended = true;
}
else
{
this.ignoredLineCount += 1;
// System.out.println("XX " + log.getRequest());
this.nextLog = log;
ended = true;
}
}
catch (Exception exception)

View file

@ -39,7 +39,7 @@ public class HttpAccessLogParser
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
public static final Pattern COMBINED_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-fA-F0-9\\:\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>.*)\".*$");
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>.*)\".*$");
/**
* Instantiates a new http access log parser.

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2022-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -34,8 +34,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogs.class);
private Files source;
private String pattern;
private String pathFilter;
private String regex;
/**
* Instantiates a new http access logs.
@ -47,7 +46,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
*/
public HttpAccessLogs(final Files source) throws IOException
{
this(source, null, null);
this(source, null);
}
/**
@ -58,28 +57,10 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogs(final Files source, final String pattern) throws IOException
{
this(source, pattern, null);
}
/**
* Instantiates a new http access logs.
*
* @param source
* the source
* @param pattern
* the pattern
* @param pathFilter
* the path filter
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogs(final Files source, final String pattern, final String pathFilter) throws IOException
public HttpAccessLogs(final Files source, final String regex) throws IOException
{
this.source = source;
this.pattern = pattern;
this.pathFilter = pathFilter;
this.regex = regex;
}
/* (non-Javadoc)
@ -92,7 +73,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
try
{
result = new HttpAccessLogIterator(this.source, this.pattern, this.pathFilter);
result = new HttpAccessLogIterator(this.source, this.regex);
}
catch (IOException exception)
{

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2022-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -31,6 +31,7 @@ import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class LibreQRProber.
@ -49,8 +50,10 @@ public class LibreQRProber
/**
* Probe.
*
* @param httpAccessLogs
* the http access logs
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @return the path counters
@ -59,7 +62,7 @@ public class LibreQRProber
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final HttpAccessLogs httpAccessLogs, final File dataPath) throws IOException, StatoolInfosException
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
{
PathCounters result;
@ -68,7 +71,7 @@ public class LibreQRProber
// metrics.service.users.ipv6
// metrics.barcodes.created
// metrics.libreqr.barcodes.downloads
result = probeHttpAccessLog(httpAccessLogs);
result = probeHttpAccessLog(httpLogs, httpLogRegex);
// metrics.service.files.bytes, metrics.libreqr.cache.bytes
// metrics.service.files.count, metrics.libreqr.cache.count
@ -79,17 +82,19 @@ public class LibreQRProber
}
/**
* Probe http access log.
* Probe.
*
* @param logs
* the logs
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
private static PathCounters probeHttpAccessLog(final HttpAccessLogs logs) throws IOException, StatoolInfosException
private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
@ -104,7 +109,7 @@ public class LibreQRProber
Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
//
for (HttpAccessLog log : logs)
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
{
// General HTTP access logs.
String year = log.getYear();

View file

@ -1,95 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.nextcloud;
import java.io.File;
import java.io.IOException;
import java.time.LocalDate;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.TimeMark;
import fr.devinsy.statoolinfos.util.FilesUtils;
// TODO: Auto-generated Javadoc
/**
* The Class LibreQRDataAnalyzer.
*/
public class NextcloudDataAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(NextcloudDataAnalyzer.class);
/**
* Instantiates a new http access log prober.
*/
private NextcloudDataAnalyzer()
{
}
/**
* Probe.
*
* @param dataDirectory
* the data directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
{
PathCounters result;
System.out.println("Probing directory [" + dataDirectory + "]");
result = new PathCounters();
if (dataDirectory != null)
{
if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
{
LocalDate now = LocalDate.now();
String year = TimeMark.yearOf(now).toString();
String yearMonth = TimeMark.yearMonthOf(now).toString();
String yearWeek = TimeMark.yearWeekOf(now).toString();
String date = TimeMark.dayOf(now).toString();
// metrics.service.files.bytes
long size = FileUtils.sizeOfDirectory(dataDirectory);
result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
// metrics.pastebins.count
long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
result.set(count, "metrics.libreqr.files.count", year, yearMonth, yearWeek, date);
}
else
{
System.out.println("WARNING: LibreQR data path is not valid.");
}
}
//
return result;
}
}

View file

@ -1,153 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.nextcloud;
import java.io.IOException;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class NextcloudHttpLogAnalyzer.
*/
public class NextcloudHttpLogAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(NextcloudHttpLogAnalyzer.class);
public static final Pattern USE_PATTERN = Pattern.compile("GET /temp/\\w+\\.png.*");
public static final Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
private PathCounters counters;
private UserCounters users;
private UserCounters ipv4Users;
private UserCounters ipv6Users;
/**
* Instantiates a new nextcloud http log analyzer.
*/
public NextcloudHttpLogAnalyzer()
{
this.counters = new PathCounters();
this.users = new UserCounters();
this.ipv4Users = new UserCounters();
this.ipv6Users = new UserCounters();
}
/**
* Gets the counters.
*
* @return the counters
*/
public PathCounters getCounters()
{
PathCounters result;
result = new PathCounters();
result.putAll(this.counters);
result.putAll(this.users.getCounters("metrics.service.users"));
result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
/**
* Probe log.
*
* @param log
* the log
*/
public void probeLog(final HttpAccessLog log)
{
if (log != null)
{
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
this.users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.barcodes.count
if ((log.getStatus().getCode() == 200) && (CREATE_PATTERN.matcher(log.getRequest()).matches()))
{
this.counters.inc("metrics.barcodes.count", year, yearMonth, yearWeek, date);
}
}
}
/**
* Probe.
*
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
NextcloudHttpLogAnalyzer analyzer = new NextcloudHttpLogAnalyzer();
HttpAccessLogIterator logs = new HttpAccessLogIterator(httpAccessLogFiles, httpRegex);
while (logs.hasNext())
{
analyzer.probeLog(logs.next());
}
result = analyzer.getCounters();
//
return result;
}
}

View file

@ -1,75 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.nextcloud;
import java.io.File;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
* The Class NextcloudProber.
*/
public class NextcloudProber
{
private static Logger logger = LoggerFactory.getLogger(NextcloudProber.class);
/**
* Instantiates a new nextcloud prober.
*/
public NextcloudProber()
{
}
/**
* Probe.
*
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
{
PathCounters result;
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
result = NextcloudHttpLogAnalyzer.probe(FilesUtils.searchByWildcard(httpLogs), httpLogRegex);
// metrics.service.files.bytes
result.putAll(NextcloudDataAnalyzer.probe(dataPath));
//
return result;
}
}

View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021-2023 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -32,6 +32,7 @@ import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
@ -51,15 +52,17 @@ public class PrivatebinProber
/**
* Probe.
*
* @param logs
* the logs
* @param httpAccessLogs
* the http access logs
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
private static PathCounters probe(final HttpAccessLogs logs) throws IOException, StatoolInfosException
private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
@ -71,7 +74,7 @@ public class PrivatebinProber
UserCounters ipv6Users = new UserCounters();
//
for (HttpAccessLog log : logs)
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogs, httpRegex))
{
// logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString());
@ -132,17 +135,19 @@ public class PrivatebinProber
/**
* Probe.
*
* @param httpAccessLogs
* the http access logs
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param datafileDirectory
* the datafile directory
* the data path
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final HttpAccessLogs httpAccessLogs, final File datafileDirectory) throws IOException, StatoolInfosException
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File datafileDirectory) throws IOException, StatoolInfosException
{
PathCounters result;
@ -152,7 +157,7 @@ public class PrivatebinProber
// metrics.pastebins.created
// metrics.pastebins.reads
// metrics.pastebins.deleted
result = probe(httpAccessLogs);
result = probe(httpLogs, httpLogRegex);
// metrics.service.files.bytes
// metrics.service.files.count

View file

@ -1,95 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.xmpp;
import java.io.File;
import java.io.IOException;
import java.time.LocalDate;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.TimeMark;
import fr.devinsy.statoolinfos.util.FilesUtils;
// TODO: Auto-generated Javadoc
/**
* The Class LibreQRDataAnalyzer.
*/
public class XmppDataAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(XmppDataAnalyzer.class);
/**
* Instantiates a new http access log prober.
*/
private XmppDataAnalyzer()
{
}
/**
* Probe.
*
* @param dataDirectory
* the data directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
{
PathCounters result;
System.out.println("Probing directory [" + dataDirectory + "]");
result = new PathCounters();
if (dataDirectory != null)
{
if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
{
LocalDate now = LocalDate.now();
String year = TimeMark.yearOf(now).toString();
String yearMonth = TimeMark.yearMonthOf(now).toString();
String yearWeek = TimeMark.yearWeekOf(now).toString();
String date = TimeMark.dayOf(now).toString();
// metrics.service.files.bytes
long size = FileUtils.sizeOfDirectory(dataDirectory);
result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
// metrics.pastebins.count
long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
result.set(count, "metrics.libreqr.files.count", year, yearMonth, yearWeek, date);
}
else
{
System.out.println("WARNING: LibreQR data path is not valid.");
}
}
//
return result;
}
}

View file

@ -1,153 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.xmpp;
import java.io.IOException;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class LibreQRHttpLogAnalyzer.
*/
public class XmppHttpLogAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(XmppHttpLogAnalyzer.class);
public static final Pattern USE_PATTERN = Pattern.compile("GET /temp/\\w+\\.png.*");
public static final Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
private PathCounters counters;
private UserCounters users;
private UserCounters ipv4Users;
private UserCounters ipv6Users;
/**
* Instantiates a new http access log prober.
*/
public XmppHttpLogAnalyzer()
{
this.counters = new PathCounters();
this.users = new UserCounters();
this.ipv4Users = new UserCounters();
this.ipv6Users = new UserCounters();
}
/**
* Gets the counters.
*
* @return the counters
*/
public PathCounters getCounters()
{
PathCounters result;
result = new PathCounters();
result.putAll(this.counters);
result.putAll(this.users.getCounters("metrics.service.users"));
result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
/**
* Probe log.
*
* @param log
* the log
*/
public void probeLog(final HttpAccessLog log)
{
if (log != null)
{
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
this.users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.barcodes.count
if ((log.getStatus().getCode() == 200) && (CREATE_PATTERN.matcher(log.getRequest()).matches()))
{
this.counters.inc("metrics.barcodes.count", year, yearMonth, yearWeek, date);
}
}
}
/**
* Probe.
*
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
XmppHttpLogAnalyzer analyzer = new XmppHttpLogAnalyzer();
HttpAccessLogIterator logs = new HttpAccessLogIterator(httpAccessLogFiles, httpRegex);
while (logs.hasNext())
{
analyzer.probeLog(logs.next());
}
result = analyzer.getCounters();
//
return result;
}
}

View file

@ -1,75 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.xmpp;
import java.io.File;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
* The Class XmppProber.
*/
public class XmppProber
{
private static Logger logger = LoggerFactory.getLogger(XmppProber.class);
/**
* Instantiates a new xmpp prober.
*/
public XmppProber()
{
}
/**
* Probe.
*
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
{
PathCounters result;
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
result = XmppHttpLogAnalyzer.probe(FilesUtils.searchByWildcard(httpLogs), httpLogRegex);
// metrics.service.files.bytes
result.putAll(XmppDataAnalyzer.probe(dataPath));
//
return result;
}
}