statool/statoolinfos
statool/statoolinfos
1
0
Fork 0
Code Issues Pull requests Releases 8 Activity

Compare commits

base: statool:205bd54b01884023d66a367f7a3bc52fa1065344
Branches Tags
statool:main
statool:0.6.1
statool:0.6.0
statool:SPLITWEB
statool:0.5.2
statool:0.5.1
statool:statoolinfos-0.5.1-SNAPSHOT_20230131174124
statool:0.5.1-SNAPSHOT_20220308125604
statool:0.5.0
statool:0.4.1
statool:0.4.0
statool:0.3.0
statool:0.1.0
..
compare: statool:20a58e2b1d103c339d981017bbfe64b8a7b373d2
Branches Tags
statool:main
statool:0.6.1
statool:0.6.0
statool:SPLITWEB
statool:0.5.2
statool:0.5.1
statool:statoolinfos-0.5.1-SNAPSHOT_20230131174124
statool:0.5.1-SNAPSHOT_20220308125604
statool:0.5.0
statool:0.4.1
statool:0.4.0
statool:0.3.0
statool:0.1.0

No commits in common. "205bd54b01884023d66a367f7a3bc52fa1065344" and "20a58e2b1d103c339d981017bbfe64b8a7b373d2" have entirely different histories.
205bd54b01 ... 20a58e2b1d

44 changed files with 491 additions and 2283 deletions
Show stats Expand all files Collapse all files

1
.classpath
View file

@ -28,6 +28,5 @@
<classpathentry kind="lib" path="lib/Logs/log4j-core-2.17.1.jar" sourcepath="lib/Logs/log4j-core-2.17.1-sources.jar"/> <classpathentry kind="lib" path="lib/Logs/log4j-core-2.17.1.jar" sourcepath="lib/Logs/log4j-core-2.17.1-sources.jar"/>
<classpathentry kind="lib" path="lib/Logs/log4j-slf4j-impl-2.17.1.jar" sourcepath="lib/Logs/log4j-slf4j-impl-2.17.1-sources.jar"/> <classpathentry kind="lib" path="lib/Logs/log4j-slf4j-impl-2.17.1.jar" sourcepath="lib/Logs/log4j-slf4j-impl-2.17.1-sources.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.11.0.jar" sourcepath="lib/commons-io-2.11.0-sources.jar"/> <classpathentry kind="lib" path="lib/commons-io-2.11.0.jar" sourcepath="lib/commons-io-2.11.0-sources.jar"/>
<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar" sourcepath="lib/json-simple-1.1.1-sources.jar"/>
<classpathentry kind="output" path="bin"/> <classpathentry kind="output" path="bin"/>
</classpath> </classpath>

4
CONCEPT.md
View file

@ -65,8 +65,8 @@ Voici une liste de types simples :
| Type | Description | Modèle | Exemple | | Type | Description | Modèle | Exemple |
| ------ | ------ | ------ | ------ | | ------ | ------ | ------ | ------ |
| DATE | Une date au format ISO | ISO_8601 | 2020-07-06 | | DATE | Une date au format ISO | ISO_DATE | 2020-07-06 |
| DATETIME | Une date horaire au format ISO | ISO_8601 | 2020-07-06T14:23:20 | | DATETIME | Une date horaire au format ISO | ISO_DATETIME | 2020-07-06T14:23:20 |
| DAYS | Une liste de 366 valeurs numériques avec séparateurs | STU | 1;2;3;4;5;…;365 | | DAYS | Une liste de 366 valeurs numériques avec séparateurs | STU | 1;2;3;4;5;…;365 |
| EMAIL | Un courriel. | ^.+@.+$ | christian@momon.org | | EMAIL | Un courriel. | ^.+@.+$ | christian@momon.org |
| MONTHS | Une liste de 12 valeurs numériques avec séparateurs | STU | 1;2;3;4;5;6;7;9;10;11;12 | | MONTHS | Une liste de 12 valeurs numériques avec séparateurs | STU | 1;2;3;4;5;6;7;9;10;11;12 |

2
ONTOLOGY.md
View file

@ -9,7 +9,7 @@ Cette section regroupe des informations sur le fichier *properties*.
file.class = federation file.class = federation
# Version de l'ontologie utilisée utilisé (type STRING, recommandé). # Version de l'ontologie utilisée utilisé (type STRING, recommandé).
file.protocol = ChatonsInfos-0.5 file.protocol = ChatonsInfos-0.1
# Date et horaire de génération du fichier (type DATETIME, recommandé). # Date et horaire de génération du fichier (type DATETIME, recommandé).
file.datetime = 2020-07-06T14:23:20 file.datetime = 2020-07-06T14:23:20

11
README.md
View file

@ -164,7 +164,6 @@ Configuration template:
``` ```
conf.probe.types=PrivateBin conf.probe.types=PrivateBin
conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log* conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
conf.probe.httpaccesslog.pattern=
conf.probe.privatebin.data=/var/www/paste.libre-service.eu/data/ conf.probe.privatebin.data=/var/www/paste.libre-service.eu/data/
conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
``` ```
@ -177,18 +176,12 @@ Configuration template:
conf.probe.types=Framadate conf.probe.types=Framadate
conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
``` ```
### Gitea metrics (coming soon)
### Gitea metrics
Configuration template: Configuration template:
``` ```
conf.probe.types=Gitea conf.probe.types=Gitea
conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
conf.probe.httpaccesslog.pattern=
conf.probe.gitea.data=/opt/gitea/data
conf.probe.gitea.api.url=https://forge.libre-service.eu/
conf.probe.gitea.token=b6598c616b1cd350b834258205da4e5e8b951005
conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
``` ```
@ -199,7 +192,6 @@ Configuration template:
``` ```
conf.probe.types=LibreQR conf.probe.types=LibreQR
conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log* conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
conf.probe.httpaccesslog.pattern=
conf.probe.libreqr.data=/var/www/qrcode.libre-service.eu/data/ conf.probe.libreqr.data=/var/www/qrcode.libre-service.eu/data/
conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
``` ```
@ -211,7 +203,6 @@ Configuration template:
``` ```
conf.probe.types=Nextcloud conf.probe.types=Nextcloud
conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log* conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
conf.probe.httpaccesslog.pattern=
conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
``` ```

BIN
lib/json-simple-1.1.1-sources.jar
View file

Binary file not shown.

BIN
lib/json-simple-1.1.1.jar
View file

Binary file not shown.

2
resources/conf/sample-web.conf
View file

@ -4,7 +4,7 @@
# Warning: move this file outside the Git directory. # Warning: move this file outside the Git directory.
conf.class=federation conf.class=federation
conf.protocol=StatoolInfos-0.5.0 conf.protocol=StatoolInfos-0.4.0
conf.crawl.input=https://www.libre-service.eu/.well-known/statoolinfos/libre-service.eu.properties conf.crawl.input=https://www.libre-service.eu/.well-known/statoolinfos/libre-service.eu.properties
conf.crawl.cache=/srv/statoolinfos/cache/ conf.crawl.cache=/srv/statoolinfos/cache/

2
snapshot
View file

@ -1,5 +1,3 @@
#!/bin/bash #!/bin/bash
cd $(dirname "$0")
./build.sh -snapshot ./build.sh -snapshot

43
src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -197,30 +197,6 @@ public final class StatoolInfosCLI
return result; return result;
} }
/**
* Normalize bash parameter.
*
* @param value
* the value
* @return the string
*/
private static String normalizeBashParameter(final String value)
{
String result;
if (value == null)
{
result = null;
}
else
{
result = StringUtils.trim(value).replace("\\*", "*").replace("\\?", "?");
}
//
return result;
}
/** /**
* Parses the log filter option. * Parses the log filter option.
* *
@ -532,23 +508,6 @@ public final class StatoolInfosCLI
} }
System.out.println(chrono.format()); System.out.println(chrono.format());
} }
else if (isMatching(args, "testHttpAccessLog", "\\s*\\S+\\s*"))
{
System.out.println("param1=" + args[1]);
String source = normalizeBashParameter(args[1]);
System.out.println("source=" + source);
Files files = FilesUtils.searchByWildcard(source);
System.out.println(files.size());
try
{
StatoolInfos.testHttpAccessLog(files);
}
catch (Exception exception)
{
logger.error("Error: {}", exception.getMessage());
exception.printStackTrace();
}
}
else else
{ {
System.out.println("Bad usage."); System.out.println("Bad usage.");

45
src/fr/devinsy/statoolinfos/core/Federation.java
View file

@ -135,21 +135,6 @@ public class Federation extends PathPropertyList
return result; return result;
} }
/**
* Gets the away organizations.
*
* @return the away organizations
*/
public Organizations getAwayOrganizations()
{
Organizations result;
result = this.organizations.filterAwayFor(getTechnicalName());
//
return result;
}
/** /**
* Gets the contact email. * Gets the contact email.
* *
@ -235,21 +220,6 @@ public class Federation extends PathPropertyList
return result; return result;
} }
/**
* Gets the idle organizations.
*
* @return the idle organizations
*/
public Organizations getIdleOrganizations()
{
Organizations result;
result = this.organizations.getIdles();
//
return result;
}
public PropertyChecks getInputChecks() public PropertyChecks getInputChecks()
{ {
return this.inputChecks; return this.inputChecks;
@ -369,21 +339,6 @@ public class Federation extends PathPropertyList
return result; return result;
} }
/**
* Gets the member organizations.
*
* @return the member organizations
*/
public Organizations getMemberOrganizations()
{
Organizations result;
result = this.organizations.filterMemberFor(getTechnicalName());
//
return result;
}
/** /**
* Gets the metric month values all. * Gets the metric month values all.
* *

67
src/fr/devinsy/statoolinfos/core/Organization.java
View file

@ -86,28 +86,6 @@ public class Organization extends PathPropertyList
this.crawlJournal = new CrawlJournal(); this.crawlJournal = new CrawlJournal();
} }
/**
* Gets the service active count.
*
* @return the service active count
*/
public int getActiveServiceCount()
{
int result;
result = 0;
for (Service service : this.services)
{
if (service.isActive())
{
result += 1;
}
}
//
return result;
}
/** /**
* Gets the active services. * Gets the active services.
* *
@ -131,6 +109,28 @@ public class Organization extends PathPropertyList
return result; return result;
} }
/**
* Gets the service active count.
*
* @return the service active count
*/
public int getActiveServiceCount()
{
int result;
result = 0;
for (Service service : this.services)
{
if (service.isActive())
{
result += 1;
}
}
//
return result;
}
/** /**
* Gets the age. * Gets the age.
* *
@ -1074,29 +1074,6 @@ public class Organization extends PathPropertyList
return result; return result;
} }
/**
* Checks if is idle.
*
* @return true, if is idle
*/
public boolean isIdle()
{
boolean result;
Status status = getStatus();
if ((status == null) || (status == Status.IDLE))
{
result = true;
}
else
{
result = false;
}
//
return result;
}
/** /**
* Checks if is valid. * Checks if is valid.
* *

98
src/fr/devinsy/statoolinfos/core/Organizations.java
View file

@ -36,29 +36,6 @@ public class Organizations extends ArrayList<Organization>
super(); super();
} }
/**
* Filter active.
*
* @return the organizations
*/
public Organizations filterActive()
{
Organizations result;
result = new Organizations();
for (Organization organization : this)
{
if (organization.isActive())
{
result.add(organization);
}
}
//
return result;
}
/** /**
* Filter member of. * Filter member of.
* *
@ -74,32 +51,7 @@ public class Organizations extends ArrayList<Organization>
for (Organization organization : this) for (Organization organization : this)
{ {
if ((organization.isActive()) && (!organization.isAwayFor(entityName))) if ((!organization.isAway()) && (!organization.isAwayFor(entityName)))
{
result.add(organization);
}
}
//
return result;
}
/**
* Filter away for.
*
* @param entityName
* the entity name
* @return the organizations
*/
public Organizations filterAwayFor(final String entityName)
{
Organizations result;
result = new Organizations();
for (Organization organization : this)
{
if ((organization.isAway()) || (organization.isAwayFor(entityName)))
{ {
result.add(organization); result.add(organization);
} }
@ -157,31 +109,6 @@ public class Organizations extends ArrayList<Organization>
return result; return result;
} }
/**
* Filter member for.
*
* @param entityName
* the entity name
* @return the organizations
*/
public Organizations filterMemberFor(final String entityName)
{
Organizations result;
result = new Organizations();
for (Organization organization : this)
{
if ((!organization.isAway()) && (!organization.isAwayFor(entityName)))
{
result.add(organization);
}
}
//
return result;
}
/** /**
* Gets the active service count. * Gets the active service count.
* *
@ -201,29 +128,6 @@ public class Organizations extends ArrayList<Organization>
return result; return result;
} }
/**
* Gets the idles.
*
* @return the idles
*/
public Organizations getIdles()
{
Organizations result;
result = new Organizations();
for (Organization organization : this)
{
if (organization.isIdle())
{
result.add(organization);
}
}
//
return result;
}
/** /**
* Gets the service count. * Gets the service count.
* *

34
src/fr/devinsy/statoolinfos/core/StatoolInfos.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -34,8 +34,7 @@ import fr.devinsy.statoolinfos.crawl.Crawler;
import fr.devinsy.statoolinfos.htmlize.Htmlizer; import fr.devinsy.statoolinfos.htmlize.Htmlizer;
import fr.devinsy.statoolinfos.metrics.Prober; import fr.devinsy.statoolinfos.metrics.Prober;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogParser;
import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathProperties;
import fr.devinsy.statoolinfos.properties.PathPropertyUtils; import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
import fr.devinsy.statoolinfos.stats.ip.IpStat; import fr.devinsy.statoolinfos.stats.ip.IpStat;
@ -47,7 +46,6 @@ import fr.devinsy.statoolinfos.stats.visitor.VisitorStator;
import fr.devinsy.statoolinfos.uptime.UptimeJournal; import fr.devinsy.statoolinfos.uptime.UptimeJournal;
import fr.devinsy.statoolinfos.uptime.UptimeSurveyor; import fr.devinsy.statoolinfos.uptime.UptimeSurveyor;
import fr.devinsy.statoolinfos.util.Chrono; import fr.devinsy.statoolinfos.util.Chrono;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.LineIterator; import fr.devinsy.statoolinfos.util.LineIterator;
import fr.devinsy.strings.StringList; import fr.devinsy.strings.StringList;
import fr.devinsy.strings.StringsUtils; import fr.devinsy.strings.StringsUtils;
@ -182,7 +180,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
@ -232,7 +230,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
@ -283,7 +281,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
@ -349,7 +347,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
stator.putLog(log); stator.putLog(log);
@ -400,7 +398,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
stator.putLog(log); stator.putLog(log);
@ -451,7 +449,7 @@ public class StatoolInfos
while (iterator.hasNext()) while (iterator.hasNext())
{ {
String line = iterator.next(); String line = iterator.next();
HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
if (filter.matches(log)) if (filter.matches(log))
{ {
stator.putLog(log); stator.putLog(log);
@ -533,22 +531,6 @@ public class StatoolInfos
} }
} }
/**
* Test http access log.
*
* @param source
* the source
* @throws IOException
*/
public static void testHttpAccessLog(final Files source) throws IOException
{
HttpAccessLogIterator logs = new HttpAccessLogIterator(source);
while (logs.hasNext())
{
System.out.println(logs.next().toStringLog());
}
}
/** /**
* Uptime. * Uptime.
* *

12
src/fr/devinsy/statoolinfos/crawl/Crawler.java
View file

@ -105,9 +105,9 @@ public class Crawler
{ {
logger.info("Crawling {}", url); logger.info("Crawling {}", url);
File downloadFile = null;
try try
{ {
File downloadFile;
try try
{ {
downloadFile = download(url); downloadFile = download(url);
@ -158,7 +158,6 @@ public class Crawler
{ {
logger.error("ERROR: bad child class [{}][{}].", downloadClass, parent); logger.error("ERROR: bad child class [{}][{}].", downloadClass, parent);
this.journal.add(url, parentURL, CrawlStatus.BADCHILDCLASS); this.journal.add(url, parentURL, CrawlStatus.BADCHILDCLASS);
downloadFile.delete();
} }
else else
{ {
@ -178,7 +177,6 @@ public class Crawler
if (StringUtils.equals(downloadSha, storedSha)) if (StringUtils.equals(downloadSha, storedSha))
{ {
this.journal.add(url, parentURL, CrawlStatus.SUCCESS); this.journal.add(url, parentURL, CrawlStatus.SUCCESS);
downloadFile.delete();
} }
else else
{ {
@ -236,13 +234,6 @@ public class Crawler
{ {
this.journal.add(url, parentURL, CrawlStatus.IOERROR); this.journal.add(url, parentURL, CrawlStatus.IOERROR);
} }
finally
{
if ((downloadFile != null) && (downloadFile.exists()))
{
downloadFile.delete();
}
}
} }
/** /**
@ -334,7 +325,6 @@ public class Crawler
{ {
final int TIMEOUT = 5000; final int TIMEOUT = 5000;
result = Files.createTempFile("tmp-", ".statoolsinfos").toFile(); result = Files.createTempFile("tmp-", ".statoolsinfos").toFile();
result.deleteOnExit();
FileUtils.copyURLToFile(url, result, TIMEOUT, TIMEOUT); FileUtils.copyURLToFile(url, result, TIMEOUT, TIMEOUT);
} }

9
src/fr/devinsy/statoolinfos/htmlize/FederationMetricGenericPage.java
View file

@ -71,16 +71,13 @@ public class FederationMetricGenericPage
// //
FederationMetricHtmlizer.htmlize(data, "graphic.users", federation, view, period, "metrics.service.users", ChartColor.GREEN); FederationMetricHtmlizer.htmlize(data, "graphic.users", federation, view, period, "metrics.service.users", ChartColor.GREEN);
FederationMetricHtmlizer.htmlize(data, "graphic.users.ipv4-ipv6", federation, view, period, "graphic.users (ipv4 + ipv6)", "metrics.service.users.ipv4", ChartColor.YELLOW, FederationMetricHtmlizer.htmlize(data, "graphic.accounts.combo", federation, view, period, "Accounts + active", "metrics.service.accounts", ChartColor.YELLOW,
"metrics.service.users.ipv6", ChartColor.GREEN); "metrics.service.accounts.active", ChartColor.GREEN);
FederationMetricHtmlizer.htmlize(data, "graphic.users.ipv4", federation, view, period, "metrics.service.users.ipv4", ChartColor.YELLOW);
FederationMetricHtmlizer.htmlize(data, "graphic.users.ipv6", federation, view, period, "metrics.service.users.ipv6", ChartColor.GREEN);
FederationMetricHtmlizer.htmlize(data, "graphic.accounts", federation, view, period, "metrics.service.accounts", ChartColor.GREEN); FederationMetricHtmlizer.htmlize(data, "graphic.accounts", federation, view, period, "metrics.service.accounts", ChartColor.GREEN);
FederationMetricHtmlizer.htmlize(data, "graphic.accounts.active", federation, view, period, "metrics.service.accounts.active", ChartColor.GREEN); FederationMetricHtmlizer.htmlize(data, "graphic.accounts.active", federation, view, period, "metrics.service.accounts.active", ChartColor.GREEN);
FederationMetricHtmlizer.htmlize(data, "graphic.database.bytes", federation, view, period, "metrics.service.database.bytes", ChartColor.GREEN); FederationMetricHtmlizer.htmlize(data, "graphic.database.bytes", federation, view, period, "metrics.service.database.bytes", ChartColor.GREEN);
FederationMetricHtmlizer.htmlize(data, "graphic.datafiles.bytes", federation, view, period, "metrics.service.datafiles.bytes", ChartColor.GREEN); FederationMetricHtmlizer.htmlize(data, "graphic.files.bytes", federation, view, period, "metrics.service.files.bytes", ChartColor.GREEN);
// //
String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/serviceMetricGenericView.xhtml", data).toString(); String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/serviceMetricGenericView.xhtml", data).toString();

3
src/fr/devinsy/statoolinfos/htmlize/FederationOrganizationsPage.java
View file

@ -78,8 +78,7 @@ public class FederationOrganizationsPage
TagDataManager data = new TagDataManager(); TagDataManager data = new TagDataManager();
data.setContent("federationHeaderView", FederationHeaderView.htmlize(federation)); data.setContent("federationHeaderView", FederationHeaderView.htmlize(federation));
data.setContent("organizationListView", OrganizationListView.htmlize(federation.getActiveOrganizations()));
data.setContent("organizationListView", OrganizationListView.htmlize(federation.getMemberOrganizations(), federation.getAwayOrganizations()));
String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/federationOrganizations.xhtml", data).toString(); String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/federationOrganizations.xhtml", data).toString();

39
src/fr/devinsy/statoolinfos/htmlize/OrganizationListView.java
View file

@ -29,7 +29,6 @@ import fr.devinsy.statoolinfos.core.Organization;
import fr.devinsy.statoolinfos.core.Organizations; import fr.devinsy.statoolinfos.core.Organizations;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.strings.StringList;
import fr.devinsy.xidyn.XidynException; import fr.devinsy.xidyn.XidynException;
import fr.devinsy.xidyn.data.TagDataManager; import fr.devinsy.xidyn.data.TagDataManager;
import fr.devinsy.xidyn.presenters.PresenterUtils; import fr.devinsy.xidyn.presenters.PresenterUtils;
@ -53,7 +52,7 @@ public class OrganizationListView
* @throws IOException * @throws IOException
* Signals that an I/O exception has occurred. * Signals that an I/O exception has occurred.
*/ */
public static String htmlize(final Organizations organizations, final Organizations aways) throws StatoolInfosException, IOException public static String htmlize(final Organizations organizations) throws StatoolInfosException, IOException
{ {
String result; String result;
@ -61,9 +60,8 @@ public class OrganizationListView
{ {
TagDataManager data = new TagDataManager(); TagDataManager data = new TagDataManager();
Organizations actives = organizations.filterActive(); data.setContent("organizationCount", organizations.size());
data.setContent("organizationCount", actives.size()); data.setContent("serviceCount", organizations.getActiveServiceCount());
data.setContent("serviceCount", actives.getActiveServiceCount());
String monthLabel = LocalDate.now().minusMonths(1).format(DateTimeFormatter.ofPattern("MMMM yyyy")).replace(" ", "&#160;"); String monthLabel = LocalDate.now().minusMonths(1).format(DateTimeFormatter.ofPattern("MMMM yyyy")).replace(" ", "&#160;");
data.setContent("monthLabel", monthLabel); data.setContent("monthLabel", monthLabel);
@ -71,7 +69,7 @@ public class OrganizationListView
data.setAttribute("visitCountHeaderColumn", "title", monthLabel); data.setAttribute("visitCountHeaderColumn", "title", monthLabel);
int index = 0; int index = 0;
for (Organization organization : actives.sortByReverseServiceCount()) for (Organization organization : organizations.sortByReverseServiceCount())
{ {
data.setAttribute("organizationListLine", index, "organizationListLineNameLink", "href", organization.getTechnicalName() + ".xhtml"); data.setAttribute("organizationListLine", index, "organizationListLineNameLink", "href", organization.getTechnicalName() + ".xhtml");
data.setAttribute("organizationListLine", index, "organizationListLineLogo", "src", organization.getLogoFileName()); data.setAttribute("organizationListLine", index, "organizationListLineLogo", "src", organization.getLogoFileName());
@ -93,35 +91,6 @@ public class OrganizationListView
index += 1; index += 1;
} }
Organizations idles = organizations.getIdles();
if (idles.isEmpty())
{
data.setContent("idleMembers", "aucun.");
}
else
{
StringList buffer = new StringList();
for (Organization idle : idles)
{
buffer.add(String.format("<a href=\"%s\" style=\"text-decoration: none;\">%s</a>", idle.getTechnicalName() + ".xhtml", idle.getName()));
}
data.setContent("idleMembers", buffer.toString("", ", ", "."));
}
if (aways.isEmpty())
{
data.setContent("awayMembers", "aucun.");
}
else
{
StringList buffer = new StringList();
for (Organization away : aways)
{
buffer.add(String.format("<a href=\"%s\" style=\"text-decoration: none;\">%s</a>", away.getTechnicalName() + ".xhtml", away.getName()));
}
data.setContent("awayMembers", buffer.toString("", ", ", "."));
}
String page = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/organizationListView.xhtml", data).toString(); String page = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/organizationListView.xhtml", data).toString();
result = XidynUtils.extractBodyContent(page); result = XidynUtils.extractBodyContent(page);

9
src/fr/devinsy/statoolinfos/htmlize/OrganizationMetricGenericPage.java
View file

@ -72,15 +72,12 @@ public class OrganizationMetricGenericPage
// //
OrganizationMetricHtmlizer.htmlize(data, "graphic.users", organization, view, period, "metrics.service.users", ChartColor.GREEN); OrganizationMetricHtmlizer.htmlize(data, "graphic.users", organization, view, period, "metrics.service.users", ChartColor.GREEN);
OrganizationMetricHtmlizer.htmlize(data, "graphic.users.ipv4-ipv6", organization, view, period, "graphic.users (ipv4 + ipv6)", "metrics.service.users.ipv4", ChartColor.YELLOW, OrganizationMetricHtmlizer.htmlize(data, "graphic.accounts.combo", organization, view, period, "Accounts + active", "metrics.service.accounts", ChartColor.YELLOW,
"metrics.service.users.ipv6", ChartColor.GREEN); "metrics.service.accounts.active", ChartColor.GREEN);
OrganizationMetricHtmlizer.htmlize(data, "graphic.users.ipv4", organization, view, period, "metrics.service.users.ipv4", ChartColor.YELLOW);
OrganizationMetricHtmlizer.htmlize(data, "graphic.users.ipv6", organization, view, period, "metrics.service.users.ipv6", ChartColor.GREEN);
OrganizationMetricHtmlizer.htmlize(data, "graphic.accounts", organization, view, period, "metrics.service.accounts", ChartColor.GREEN); OrganizationMetricHtmlizer.htmlize(data, "graphic.accounts", organization, view, period, "metrics.service.accounts", ChartColor.GREEN);
OrganizationMetricHtmlizer.htmlize(data, "graphic.accounts.active", organization, view, period, "metrics.service.accounts.active", ChartColor.GREEN); OrganizationMetricHtmlizer.htmlize(data, "graphic.accounts.active", organization, view, period, "metrics.service.accounts.active", ChartColor.GREEN);
OrganizationMetricHtmlizer.htmlize(data, "graphic.database.bytes", organization, view, period, "metrics.service.database.bytes", ChartColor.GREEN); OrganizationMetricHtmlizer.htmlize(data, "graphic.database.bytes", organization, view, period, "metrics.service.database.bytes", ChartColor.GREEN);
OrganizationMetricHtmlizer.htmlize(data, "graphic.datafiles.bytes", organization, view, period, "metrics.service.datafiles.bytes", ChartColor.GREEN); OrganizationMetricHtmlizer.htmlize(data, "graphic.files.bytes", organization, view, period, "metrics.service.files.bytes", ChartColor.GREEN);
// //
String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/serviceMetricGenericView.xhtml", data).toString(); String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/serviceMetricGenericView.xhtml", data).toString();

9
src/fr/devinsy/statoolinfos/htmlize/ServiceMetricGenericPage.java
View file

@ -72,16 +72,13 @@ public class ServiceMetricGenericPage
// //
ServiceMetricHtmlizer.htmlize(data, "graphic.users", service, view, period, "metrics.service.users", ChartColor.GREEN); ServiceMetricHtmlizer.htmlize(data, "graphic.users", service, view, period, "metrics.service.users", ChartColor.GREEN);
ServiceMetricHtmlizer.htmlize(data, "graphic.users.ipv4-ipv6", service, view, period, "graphic.users (ipv4 + ipv6)", "metrics.service.users.ipv4", ChartColor.YELLOW, ServiceMetricHtmlizer.htmlize(data, "graphic.accounts.combo", service, view, period, "Accounts + active", "metrics.service.accounts", ChartColor.YELLOW,
"metrics.service.users.ipv6", ChartColor.GREEN); "metrics.service.accounts.active", ChartColor.GREEN);
ServiceMetricHtmlizer.htmlize(data, "graphic.users.ipv4", service, view, period, "metrics.service.users.ipv4", ChartColor.YELLOW);
ServiceMetricHtmlizer.htmlize(data, "graphic.users.ipv6", service, view, period, "metrics.service.users.ipv6", ChartColor.GREEN);
ServiceMetricHtmlizer.htmlize(data, "graphic.accounts", service, view, period, "metrics.service.accounts", ChartColor.GREEN); ServiceMetricHtmlizer.htmlize(data, "graphic.accounts", service, view, period, "metrics.service.accounts", ChartColor.GREEN);
ServiceMetricHtmlizer.htmlize(data, "graphic.accounts.active", service, view, period, "metrics.service.accounts.active", ChartColor.GREEN); ServiceMetricHtmlizer.htmlize(data, "graphic.accounts.active", service, view, period, "metrics.service.accounts.active", ChartColor.GREEN);
ServiceMetricHtmlizer.htmlize(data, "graphic.database.bytes", service, view, period, "metrics.service.database.bytes", ChartColor.GREEN); ServiceMetricHtmlizer.htmlize(data, "graphic.database.bytes", service, view, period, "metrics.service.database.bytes", ChartColor.GREEN);
ServiceMetricHtmlizer.htmlize(data, "graphic.datafiles.bytes", service, view, period, "metrics.service.datafiles.bytes", ChartColor.GREEN); ServiceMetricHtmlizer.htmlize(data, "graphic.files.bytes", service, view, period, "metrics.service.files.bytes", ChartColor.GREEN);
// metrics.moderation.accounts.reported.* = // metrics.moderation.accounts.reported.* =
// metrics.moderation.accounts.sanctioned.* = // metrics.moderation.accounts.sanctioned.* =

6
src/fr/devinsy/statoolinfos/htmlize/organizationListView.xhtml
View file

@ -13,8 +13,8 @@
<body> <body>
<div class="center_table" style="width: 900px;"> <div class="center_table" style="width: 900px;">
<br/> <br/>
<div class="center">Nombre de membres actifs : <span id="organizationCount">n/a</span></div> <div class="center">Nombre de membres : <span id="organizationCount">n/a</span></div>
<div class="center">Nombre de services acfifs : <span id="serviceCount">n/a</span></div> <div class="center">Nombre de services : <span id="serviceCount">n/a</span></div>
<table id="organizations" class="table_classic left"> <table id="organizations" class="table_classic left">
<thead> <thead>
<tr> <tr>
@ -43,8 +43,6 @@
</tbody> </tbody>
</table> </table>
<div id="month_asterisk">(*) chiffres de <span id="monthLabel">n/a</span></div> <div id="month_asterisk">(*) chiffres de <span id="monthLabel">n/a</span></div>
<div id="idleMembersZone" class="left">Membres en sommeil : <span id="idleMembers">n/a</span></div>
<div id="awayMembersZone" class="left">Anciens membres : <span id="awayMembers">n/a</span></div>
</div> </div>
<script type="text/javascript"> <script type="text/javascript">
$(document).ready(function() $(document).ready(function()

6
src/fr/devinsy/statoolinfos/htmlize/serviceHeaderView.xhtml
View file

@ -28,9 +28,9 @@
Inscription : Inscription :
<img id="registrationNoneImg" src="circle-icons/global-mono.svg" class="disabled" title="Sans compte"/> <img id="registrationNoneImg" src="circle-icons/global-mono.svg" class="disabled" title="Sans compte"/>
<img id="registrationFreeImg" src="circle-icons/profile-mono.svg" class="disabled" title="Compte libre"/> <img id="registrationFreeImg" src="circle-icons/profile-mono.svg" class="disabled" title="Compte libre"/>
<img id="registrationMemberImg" src="circle-icons/rgb-mono.svg" class="disabled" title="Réservée aux membres"/> <img id="registrationMemberImg" src="circle-icons/rgb-mono.svg" class="disabled" title="Réservé aux membres"/>
<img id="registrationClientImg" src="circle-icons/creditcard-mono.svg" class="disabled" title="Réservée aux clients"/> <img id="registrationClientImg" src="circle-icons/creditcard-mono.svg" class="disabled" title="Réservé aux clients"/>
<img id="registrationLoadImg" src="circle-icons/lock-mono.svg" class="disabled" title="Actuellement fermée"/> <img id="registrationLoadImg" src="circle-icons/lock-mono.svg" class="disabled" title="Actuellement fermé"/>
</div> </div>
<div class="content_infos" style="margin: 5px;"> <div class="content_infos" style="margin: 5px;">
<a id="legalLink" href="#"><img id="legalLinkImg" src="circle-icons/ribbon.svg" class="disabled" title="Mentions légales"/>Légal</a> <a id="legalLink" href="#"><img id="legalLinkImg" src="circle-icons/ribbon.svg" class="disabled" title="Mentions légales"/>Légal</a>

6
src/fr/devinsy/statoolinfos/htmlize/serviceListView.xhtml
View file

@ -93,13 +93,13 @@
<img id="serviceListLineRegistrationFreeImg" src="circle-icons/profile-mono.svg" class="disabled" title="Compte libre"/> <img id="serviceListLineRegistrationFreeImg" src="circle-icons/profile-mono.svg" class="disabled" title="Compte libre"/>
</td> </td>
<td id="serviceListLineRegistrationMember" class="td_icon_alone"> <td id="serviceListLineRegistrationMember" class="td_icon_alone">
<img id="serviceListLineRegistrationMemberImg" src="circle-icons/rgb-mono.svg" class="disabled" title="Réservée aux membres"/> <img id="serviceListLineRegistrationMemberImg" src="circle-icons/rgb-mono.svg" class="disabled" title="Réservé aux membres"/>
</td> </td>
<td id="serviceListLineRegistrationClient" class="td_icon_alone"> <td id="serviceListLineRegistrationClient" class="td_icon_alone">
<img id="serviceListLineRegistrationClientImg" src="circle-icons/creditcard-mono.svg" class="disabled" title="Réservée aux clients"/> <img id="serviceListLineRegistrationClientImg" src="circle-icons/creditcard-mono.svg" class="disabled" title="Réservé aux clients"/>
</td> </td>
<td id="serviceListLineRegistrationLoad" class="td_icon_alone"> <td id="serviceListLineRegistrationLoad" class="td_icon_alone">
<img id="serviceListLineRegistrationLoadImg" src="circle-icons/lock-mono.svg" class="disabled" title="Actuellement fermée"/> <img id="serviceListLineRegistrationLoadImg" src="circle-icons/lock-mono.svg" class="disabled" title="Actuellement fermé"/>
</td> </td>
<td id="" class="td_icon_alone"> <td id="" class="td_icon_alone">

7
src/fr/devinsy/statoolinfos/htmlize/serviceMetricGenericView.xhtml
View file

@ -18,16 +18,13 @@
<h2>Utilisateurs</h2> <h2>Utilisateurs</h2>
<div id="graphic.users" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/> <div id="graphic.users" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
<br/> <br/>
<div id="graphic.users.ipv4-ipv6" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/> <div id="graphic.accounts.combo" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
<div id="graphic.users.ipv4" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
<div id="graphic.users.ipv6" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
<br/>
<div id="graphic.accounts" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/> <div id="graphic.accounts" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
<div id="graphic.accounts.active" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/> <div id="graphic.accounts.active" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
<h2>Données</h2> <h2>Données</h2>
<div id="graphic.database.bytes" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/> <div id="graphic.database.bytes" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
<div id="graphic.datafiles.bytes" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/> <div id="graphic.files.bytes" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
</div> </div>
</div> </div>
</body> </body>

86
src/fr/devinsy/statoolinfos/metrics/PathCounters.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -210,27 +210,6 @@ public class PathCounters extends HashMap<String, PathCounter>
return result; return result;
} }
public StringList getNowTimeMarks()
{
StringList result;
result = new StringList();
LocalDate now = LocalDate.now();
String year = TimeMark.yearOf(now).toString();
String yearMonth = TimeMark.yearMonthOf(now).toString();
String yearWeek = TimeMark.yearWeekOf(now).toString();
String date = TimeMark.dayOf(now).toString();
result.add(year);
result.add(yearMonth);
result.add(yearWeek);
result.add(date);
//
return result;
}
/** /**
* Gets the prefixes. * Gets the prefixes.
* *
@ -414,38 +393,6 @@ public class PathCounters extends HashMap<String, PathCounter>
} }
} }
/**
* Rename path.
*
* @param oldPath
* the old path
* @param newPath
* the new path
* @return the long
*/
public long renamePath(final String oldPath, final String newPath)
{
long result;
result = 0;
for (String key : new StringList(this.keySet()))
{
if (key.startsWith(oldPath))
{
result += 1;
PathCounter counter = get(key);
counter.setPath(counter.getPath().replace(oldPath, newPath));
remove(key);
put(counter);
System.out.println("Renamed " + key + " to " + counter.getPath());
}
}
//
return result;
}
/** /**
* Search by date. * Search by date.
* *
@ -536,19 +483,6 @@ public class PathCounters extends HashMap<String, PathCounter>
return result; return result;
} }
/**
* Sets the.
*
* @param value
* the value
* @param path
* the path
*/
public void set(final long value, final String path)
{
set(value, path, getNowTimeMarks());
}
/** /**
* Sets the. * Sets the.
* *
@ -588,22 +522,4 @@ public class PathCounters extends HashMap<String, PathCounter>
counter.setCounter(value); counter.setCounter(value);
} }
/**
* Sets the.
*
* @param value
* the value
* @param path
* the path
* @param timeMarks
* the time marks
*/
public void set(final long value, final String path, final StringList timeMarks)
{
for (String timeMark : timeMarks)
{
set(value, path, timeMark);
}
}
} }

68
src/fr/devinsy/statoolinfos/metrics/Prober.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -38,10 +38,8 @@ import fr.devinsy.statoolinfos.core.DatabaseConfig;
import fr.devinsy.statoolinfos.core.Factory; import fr.devinsy.statoolinfos.core.Factory;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.statoolinfos.metrics.gitea.GiteaProber;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer; import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer;
import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber;
import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber; import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber;
import fr.devinsy.statoolinfos.metrics.mumble.MumbleProber; import fr.devinsy.statoolinfos.metrics.mumble.MumbleProber;
import fr.devinsy.statoolinfos.metrics.privatebin.PrivatebinProber; import fr.devinsy.statoolinfos.metrics.privatebin.PrivatebinProber;
@ -49,7 +47,6 @@ import fr.devinsy.statoolinfos.properties.PathProperties;
import fr.devinsy.statoolinfos.properties.PathProperty; import fr.devinsy.statoolinfos.properties.PathProperty;
import fr.devinsy.statoolinfos.properties.PathPropertyUtils; import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
import fr.devinsy.statoolinfos.util.BuildInformation; import fr.devinsy.statoolinfos.util.BuildInformation;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.strings.StringList; import fr.devinsy.strings.StringList;
import fr.devinsy.strings.StringsUtils; import fr.devinsy.strings.StringsUtils;
@ -112,7 +109,7 @@ public class Prober
// //
if (types.containsAnyIgnoreCase("HttpAccessLog")) if (types.containsAnyIgnoreCase("HttpAccessLog"))
{ {
logger.info("== Probing HttpAccessLog."); logger.info("== Processing HttpAccessLog.");
String source = configuration.getProbeHttpAccessLogSource(); String source = configuration.getProbeHttpAccessLogSource();
String patternRegex = configuration.getProbeHttpAccessLogPattern(); String patternRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("source=[{}]", source); logger.info("source=[{}]", source);
@ -125,7 +122,7 @@ public class Prober
// //
if (types.containsAnyIgnoreCase("HttpErrorLog")) if (types.containsAnyIgnoreCase("HttpErrorLog"))
{ {
logger.info("== Probing HttpErrorLog."); logger.info("== Processing HttpErrorLog.");
String source = configuration.getProbeHttpErrorLogSource(); String source = configuration.getProbeHttpErrorLogSource();
logger.info("source=[{}]", source); logger.info("source=[{}]", source);
@ -136,56 +133,18 @@ public class Prober
// //
if (types.containsAnyIgnoreCase("Framadate")) if (types.containsAnyIgnoreCase("Framadate"))
{ {
logger.info("== Probing Framadate."); logger.info("== Processing Framadate.");
String source = configuration.getProbeHttpAccessLogSource(); String source = configuration.getProbeHttpErrorLogSource();
logger.info("source=[{}]", source); logger.info("source=[{}]", source);
// PathCounters datafilesPath = // PathCounters data = HttpErrorLogAnalyzer.probe(source);
// HttpErrorLogAnalyzer.probe(source);
// counters.putAll(data); // counters.putAll(data);
} }
//
if (types.containsAnyIgnoreCase("Gitea"))
{
logger.info("== Probing Gitea.");
String httpLogs = configuration.getProbeHttpAccessLogSource();
logger.info("httpLogs=[{}]", httpLogs);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("httpAccessPattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.gitea.data");
logger.info("dataPath=[{}]", dataDirectory);
String apiURL = configuration.get("conf.probe.gitea.api.url");
logger.info("apiURL=[{}]", apiURL);
String apiToken = configuration.get("conf.probe.gitea.api.token");
logger.info("apiToken=[{}]", apiToken == null ? "null" : "************************");
PathCounters metrics = GiteaProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, apiURL, apiToken, dataDirectory);
counters.putAll(metrics);
}
//
if (types.containsAnyIgnoreCase("LibreQR"))
{
logger.info("== Probing LibreQR.");
String source = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", source);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("pattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles");
logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = LibreQRProber.probe(FilesUtils.searchByWildcard(source), httpAccessLogRegex, dataDirectory);
counters.putAll(data);
}
// //
if (types.containsAnyIgnoreCase("Minetest")) if (types.containsAnyIgnoreCase("Minetest"))
{ {
logger.info("== Probing Minetest."); logger.info("== Processing Minetest.");
String source = configuration.get("conf.probe.minetest.logs"); String source = configuration.get("conf.probe.minetest.logs");
DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.minetest.players"); DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.minetest.players");
@ -196,7 +155,7 @@ public class Prober
// //
if (types.containsAnyIgnoreCase("Mumble")) if (types.containsAnyIgnoreCase("Mumble"))
{ {
logger.info("== Probing Mumble."); logger.info("== Processing Mumble.");
String source = configuration.get("conf.probe.mumble.logs"); String source = configuration.get("conf.probe.mumble.logs");
logger.info("source=[{}]", source); logger.info("source=[{}]", source);
@ -207,15 +166,15 @@ public class Prober
// //
if (types.containsAnyIgnoreCase("PrivateBin")) if (types.containsAnyIgnoreCase("PrivateBin"))
{ {
logger.info("== Probing Privatebin."); logger.info("== Processing Privatebin.");
String httpAccessLogs = configuration.getProbeHttpAccessLogSource(); String httpAccessLogs = configuration.getProbeHttpAccessLogSource();
logger.info("source=[{}]", httpAccessLogs);
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
logger.info("pattern=[{}]", httpAccessLogRegex);
File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data"); File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
logger.info("source=[{}]", httpAccessLogs);
logger.info("pattern=[{}]", httpAccessLogRegex);
logger.info("dataDirectory=[{}]", dataDirectory); logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = PrivatebinProber.probe(FilesUtils.searchByWildcard(httpAccessLogs), httpAccessLogRegex, dataDirectory); PathCounters data = PrivatebinProber.probe(httpAccessLogs, httpAccessLogRegex, dataDirectory);
counters.putAll(data); counters.putAll(data);
} }
@ -243,9 +202,6 @@ public class Prober
PathCounters previousCounters = readMetrics(target); PathCounters previousCounters = readMetrics(target);
logger.info("previous size={}", previousCounters.size()); logger.info("previous size={}", previousCounters.size());
// Temporary fixed 2022-02.
previousCounters.renamePath("metrics.service.files", "metrics.service.datafiles");
// Merge. // Merge.
logger.info("== Merging"); logger.info("== Merging");
for (PathCounter counter : counters.values()) for (PathCounter counter : counters.values())

54
src/fr/devinsy/statoolinfos/metrics/TimeMarkUtils.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -21,7 +21,6 @@ package fr.devinsy.statoolinfos.metrics;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.YearMonth; import java.time.YearMonth;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Locale; import java.util.Locale;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -219,23 +218,6 @@ public class TimeMarkUtils
return result; return result;
} }
/**
* Year month of.
*
* @param date
* the date
* @return the string
*/
public static String yearMonthOf(final ZonedDateTime date)
{
String result;
result = date.format(DateTimeFormatter.ofPattern("yyyy-MM", Locale.FRANCE));
//
return result;
}
/** /**
* Year of. * Year of.
* *
@ -253,23 +235,6 @@ public class TimeMarkUtils
return result; return result;
} }
/**
* Year of.
*
* @param date
* the date
* @return the string
*/
public static String yearOf(final ZonedDateTime date)
{
String result;
result = date.format(DateTimeFormatter.ofPattern("yyyy", Locale.FRANCE));
//
return result;
}
/** /**
* Year week of. * Year week of.
* *
@ -286,21 +251,4 @@ public class TimeMarkUtils
// //
return result; return result;
} }
/**
* Year week of.
*
* @param date
* the date
* @return the string
*/
public static String yearWeekOf(final ZonedDateTime date)
{
String result;
result = YearWeek.from(date).toString();
//
return result;
}
} }

368
src/fr/devinsy/statoolinfos/metrics/gitea/GiteaAPI.java
View file

@ -1,368 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.gitea;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class GiteaAPI.
*/
public class GiteaAPI
{
private static Logger logger = LoggerFactory.getLogger(GiteaAPI.class);
private String url;
private String token;
private JSONArray users;
private JSONArray organizations;
private JSONArray repositories;
/**
* Instantiates a new gitea API.
*/
public GiteaAPI(final String url, final String token) throws IOException, ParseException
{
this.url = StringUtils.removeEnd(url, "/");
this.token = token;
//
String json = IOUtils.toString(new URL(this.url + "/api/v1/admin/users?limit=100000&token=" + token), Charset.defaultCharset());
this.users = (JSONArray) (new JSONParser().parse(json));
//
json = IOUtils.toString(new URL(this.url + "/api/v1/admin/orgs?limit=100000&token=" + token), Charset.defaultCharset());
this.organizations = (JSONArray) (new JSONParser().parse(json));
//
json = IOUtils.toString(new URL(this.url + "/api/v1/repos/search?limit=100000&token=" + token), Charset.defaultCharset());
this.repositories = (JSONArray) ((JSONObject) (new JSONParser().parse(json))).get("data");
}
/**
* Gets the active user count.
*
* @return the active user count
*/
public long getActiveUserCount()
{
long result;
result = 0;
LocalDateTime limit = LocalDateTime.now().minusDays(31);
for (Object user : this.users)
{
String lastLogin = getJSONString(user, "last_login");
if (!StringUtils.isBlank(lastLogin))
{
try
{
LocalDateTime date = LocalDateTime.parse(lastLogin, DateTimeFormatter.ISO_DATE_TIME);
if (date.isAfter(limit))
{
result += 1;
}
}
catch (DateTimeParseException exception)
{
System.out.println("OUPS");
}
}
}
//
return result;
}
/**
* Gets the fork repository count.
*
* @return the fork repository count
*/
public long getForkRepositoryCount()
{
long result;
result = 0;
for (Object repository : this.repositories)
{
if (getJSONBoolean(repository, "fork"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the mirror repository count.
*
* @return the mirror repository count
*/
public long getMirrorRepositoryCount()
{
long result;
result = 0;
for (Object repository : this.repositories)
{
if (getJSONBoolean(repository, "mirror"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the organization count.
*
* @return the organization count
*/
public long getOrganizationCount()
{
long result;
result = this.organizations.size();
//
return result;
}
/**
* Gets the private organization count.
*
* @return the private organization count
*/
public long getPrivateOrganizationCount()
{
long result;
result = 0;
for (Object organization : this.organizations)
{
String visibility = getJSONString(organization, "visibility");
if (StringUtils.equals(visibility, "private"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the private repository count.
*
* @return the private repository count
*/
public long getPrivateRepositoryCount()
{
long result;
result = 0;
for (Object repository : this.repositories)
{
if (getJSONBoolean(repository, "private"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the public organization count.
*
* @return the public organization count
*/
public long getPublicOrganizationCount()
{
long result;
result = 0;
for (Object organization : this.organizations)
{
String visibility = getJSONString(organization, "visibility");
if (StringUtils.equals(visibility, "public"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the public repository count.
*
* @return the public repository count
*/
public long getPublicRepositoryCount()
{
long result;
result = 0;
for (Object repository : this.repositories)
{
if (!getJSONBoolean(repository, "private"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the repository count.
*
* @return the repository count
*/
public long getRepositoryCount()
{
long result;
result = this.repositories.size();
//
return result;
}
/**
* Gets the user count.
*
* @return the user count
*/
public long getUserCount()
{
long result;
result = this.users.size();
//
return result;
}
/**
* Gets the boolean.
*
* @param object
* the object
* @param key
* the key
* @return the boolean
*/
private static boolean getJSONBoolean(final Object object, final String key)
{
Boolean result;
Object value = ((JSONObject) object).get(key);
if (value == null)
{
result = false;
}
else
{
result = (Boolean) value;
}
//
return result;
}
/**
* Gets the long.
*
* @param object
* the object
* @param key
* the key
* @return the long
*/
public static long getJSONLong(final Object object, final String key)
{
long result;
Object value = ((JSONObject) object).get(key);
if (value == null)
{
result = 0;
}
else
{
result = (Long) value;
}
//
return result;
}
/**
* Gets the as string.
*
* @param object
* the object
* @param key
* the key
* @return the as string
*/
public static String getJSONString(final Object object, final String key)
{
String result;
Object value = ((JSONObject) object).get(key);
if (value == null)
{
result = "";
}
else
{
result = (String) value;
}
//
return result;
}
}

307
src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java
View file

@ -1,307 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.gitea;
import java.io.File;
import java.io.IOException;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatabaseProber;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.strings.StringList;
/**
* The Class GiteaProber.
*/
public class GiteaProber
{
private static Logger logger = LoggerFactory.getLogger(GiteaProber.class);
/**
* Instantiates a new nextcloud prober.
*/
public GiteaProber()
{
}
/**
* Probe.
*
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final String apiURL, final String apiToken, final File dataPath) throws IOException, StatoolInfosException
{
PathCounters result;
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
result = probeHttpAccessLog(httpLogs, httpLogRegex);
// metrics.service.database.bytes
result.putAll(DatabaseProber.probe(null));
// metrics.service.files.bytes
// metrics.service.files.count
result.putAll(DatafilesProber.probe(dataPath));
//
System.out.println("Probing GiteaAPI [" + apiURL + "]");
if (!StringUtils.isBlank(apiURL) && (!StringUtils.isBlank(apiToken)))
{
try
{
GiteaAPI gitea = new GiteaAPI(apiURL, apiToken);
// metrics.service.accounts
// metrics.service.accounts.active
result.set(gitea.getUserCount(), "metrics.service.accounts");
result.set(gitea.getActiveUserCount(), "metrics.service.accounts.active");
// metrics.forge.groups.* =
// metrics.forge.groups.private.* =
// metrics.forge.groups.public.* =
result.set(gitea.getOrganizationCount(), "metrics.forge.groups");
result.set(gitea.getPublicOrganizationCount(), "metrics.forge.groups.public");
result.set(gitea.getPrivateOrganizationCount(), "metrics.forge.groups.private");
// metrics.forge.projects.* =
// metrics.forge.repositories.* =
// metrics.forge.repositories.private.* =
// metrics.forge.repositories.public.* =
result.set(gitea.getRepositoryCount(), "metrics.forge.projects");
result.set(gitea.getRepositoryCount(), "metrics.forge.repositories");
result.set(gitea.getPublicRepositoryCount(), "metrics.forge.repositories.public");
result.set(gitea.getPrivateRepositoryCount(), "metrics.forge.repositories.private");
result.set(gitea.getForkRepositoryCount(), "metrics.forge.repositories.forks");
result.set(gitea.getMirrorRepositoryCount(), "metrics.forge.repositories.mirrors");
// metrics.issues.count.* =
// metrics.issues.issuers.*=
// metrics.issues.created.* =
// metrics.issues.closed.* =
}
catch (IOException | ParseException exception)
{
exception.printStackTrace();
}
}
// ///////////////////////////////////////
// # [Metrics spécifiques aux services de forges logicielles].
//
//
// metrics.forge.commits.* =
// metrics.forge.mergerequests.* =
// metrics.forge.committers.* =
// metrics.forge.mergerequesters.* =
// metrics.forge.files.* =
//
return result;
}
/**
* Probe http access log.
*
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
result = new PathCounters();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
UserCounters users = new UserCounters();
UserCounters ipv4Users = new UserCounters();
UserCounters ipv6Users = new UserCounters();
StringList regexList = new StringList();
/*
GET /admin
GET /admin/auths
GET /admin/config
GET /admin/emails
GET /admin/hooks
GET /admin/monitor
GET /admin/notices
GET /admin/orgs
GET /admin/repos
GET /admin/users
*/
regexList.add("GET /admin(/auths|/config|/emails|/hooks|/monitor|/notices|/orgs|/repos|/users)? .*");
/*
GET /cpm
GET /devinsy
*/
/*
GET /devinsy/bacasable
GET /devinsy/bacasable/activity
GET /devinsy/bacasable/commits/branch/master
GET /devinsy/bacasable/compare/master...master
GET /devinsy/bacasable/graph
GET /devinsy/bacasable/issues
GET /devinsy/bacasable/issues/new?project=1
GET /devinsy/bacasable/projects
GET /devinsy/bacasable/projects/1
GET /devinsy/bacasable/projects/1/edit
GET /devinsy/bacasable/pulls
GET /devinsy/bacasable/pulls/1
GET /devinsy/bacasable/releases
GET /devinsy/bacasable/releases/tag/0.5.0
GET /devinsy/bacasable/tags
GET /devinsy/bacasable/wiki
GET /devinsy/bacasable/wiki/Home
*/
regexList.add("GET /\\S+/\\S+(/activity|/commits/branch/master|/compare/\\S+|/graph|/issues|/projects(/\\d+)|/pulls(/\\d+)|/releases(/tag/\\S+|/tags|/wiki|/wiki/\\S+)) .*");
/*
GET /devinsy/bacasable/settings
GET /devinsy/bacasable/settings/branches
GET /devinsy/bacasable/settings/collaboration
GET /devinsy/bacasable/settings/hooks
GET /devinsy/bacasable/settings/keys
GET /devinsy/bacasable/settings/lfs
GET /devinsy/bacasable/settings/lfs/locks
GET /devinsy/bacasable/settings/lfs/pointers
GET /devinsy/bacasable/settings/tags
*/
regexList.add("GET /\\S+/\\S+/settings(/branches|/collaboration|/hooks|/keys|/lfs|/lfs/locks|/lfs/pointers|/tags)? .*");
/*
GET /org/create
GET /org/devinsy/members
GET /org/devinsy/settings
GET /org/devinsy/teams
GET /org/devinsy/teams/new
*/
regexList.add("GET /org/(create|/\\S+/members|/\\S+/settings|/\\S+/teams|/\\S+/teams/new) .*");
/*
GET /explore/organizations
GET /explore/repos
GET /explore/users
*/
regexList.add("GET (/explore/organizations|/explore/repos|/explore/users) .*");
/*
GET /issues
GET /milestones
GET /pulls
*/
regexList.add("GET /(issues|milestones|pulls) .*");
/*
GET /repo/create
GET /repo/fork/7
GET /repo/migrate
*/
regexList.add("GET /repo/(create|fork/\\d+|migrate) .*");
/*
GET /user/settings
GET /user/settings/account
GET /user/settings/appearance
GET /user/settings/applications
GET /user/settings/keys
GET /user/settings/organization
GET /user/settings/repos
GET /user/settings/security
*/
regexList.add("GET /user/settings(/account|/appearance|/applications|/keys|/organizations|/repos|/security)? .*");
String regex = regexList.toString("(", "|", ")");
Pattern USE_PATTERN = Pattern.compile(regex);
//
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
{
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
}
//
result.putAll(users.getCounters("metrics.service.users"));
result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
}

49
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -19,8 +19,6 @@
package fr.devinsy.statoolinfos.metrics.http; package fr.devinsy.statoolinfos.metrics.http;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Locale; import java.util.Locale;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -42,7 +40,7 @@ public class HttpAccessLog
private String ip; private String ip;
private String remoteUser; private String remoteUser;
private ZonedDateTime time; private LocalDateTime time;
private String request; private String request;
private HttpStatus status; private HttpStatus status;
private long bodyBytesSent; private long bodyBytesSent;
@ -136,7 +134,7 @@ public class HttpAccessLog
public LocalDateTime getTime() public LocalDateTime getTime()
{ {
return this.time.toLocalDateTime(); return this.time;
} }
public UserAgent getUserAgent() public UserAgent getUserAgent()
@ -319,18 +317,7 @@ public class HttpAccessLog
this.status = status; this.status = status;
} }
/**
* Sets the time.
*
* @param time
* the new time
*/
public void setTime(final LocalDateTime time) public void setTime(final LocalDateTime time)
{
this.time = ZonedDateTime.of(time, ZoneId.systemDefault());
}
public void setTime(final ZonedDateTime time)
{ {
this.time = time; this.time = time;
} }
@ -366,34 +353,4 @@ public class HttpAccessLog
// //
return result; return result;
} }
/**
* To string log.
*
* @return the string
*/
public String toStringLog()
{
String result;
// "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+)
// \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+)
// (?<bodyBytesSent>\\d+) \"(?<referer>.*)\"
// \"(?<userAgent>[^\"]*)\".*$");
// result = String.format("%s %s %s \\[%s\\] \"%s\" %d %d \"%s\"
// \"%s\"",
result = String.format("%s - %s [%s] \"%s\" %d %d \"%s\" \"%s\"",
this.ip,
this.remoteUser,
this.time.format(DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH)),
this.request,
this.status.getCode(),
this.bodyBytesSent,
this.referer,
this.userAgent);
//
return result;
}
} }

46
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java
View file

@ -20,6 +20,9 @@ package fr.devinsy.statoolinfos.metrics.http;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -40,6 +43,12 @@ public class HttpAccessLogAnalyzer
{ {
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogAnalyzer.class); private static Logger logger = LoggerFactory.getLogger(HttpAccessLogAnalyzer.class);
// log_format combined '$remote_addr - $remote_user [$time_local] '
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
public static final Pattern COMBINED_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>.*)\" \"(?<userAgent>[^\"]*)\".*$");
private int errorCount; private int errorCount;
private PathCounters counters; private PathCounters counters;
private IpCounters ips; private IpCounters ips;
@ -140,7 +149,7 @@ public class HttpAccessLogAnalyzer
Pattern pattern; Pattern pattern;
if (patternRegex == null) if (patternRegex == null)
{ {
pattern = HttpAccessLogParser.COMBINED_PATTERN; pattern = COMBINED_PATTERN;
} }
else else
{ {
@ -155,7 +164,7 @@ public class HttpAccessLogAnalyzer
try try
{ {
HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern); HttpAccessLog log = parseLog(line, pattern);
if (log == null) if (log == null)
{ {
logger.warn("LINE IS NOT MATCHING [{}]", line); logger.warn("LINE IS NOT MATCHING [{}]", line);
@ -401,6 +410,39 @@ public class HttpAccessLogAnalyzer
return result; return result;
} }
/**
* Parses the log.
*
* @param line
* the line
* @return the http log
*/
public static HttpAccessLog parseLog(final String line, final Pattern pattern)
{
HttpAccessLog result;
Matcher matcher = pattern.matcher(line);
if (matcher.matches())
{
result = new HttpAccessLog();
result.setIp(matcher.group("remoteAddress"));
result.setRemoteUser(matcher.group("remoteUser"));
result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
result.setRequest(matcher.group("request"));
result.setStatus(HttpStatusTable.instance().get(matcher.group("status")));
result.setBodyBytesSent(Long.valueOf(matcher.group("bodyBytesSent")));
result.setReferer(matcher.group("referer"));
result.setUserAgent(new UserAgent(matcher.group("userAgent")));
}
else
{
result = null;
}
//
return result;
}
/** /**
* Probe. * Probe.
* *

159
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java
View file

@ -1,159 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.http;
import java.io.IOException;
import java.util.Iterator;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.FilesLineIterator;
/**
* The Class HttpAccessLogIterator.
*/
public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogIterator.class);
private FilesLineIterator lineIterator;
private Pattern pattern;
private HttpAccessLog nextLog;
/**
* Instantiates a new http access log iterator.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogIterator(final Files source) throws IOException
{
this(source, null);
}
/**
* Instantiates a new http log iterator.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogIterator(final Files source, final String regex) throws IOException
{
this.lineIterator = new FilesLineIterator(source);
this.nextLog = null;
if (regex == null)
{
this.pattern = HttpAccessLogParser.COMBINED_PATTERN;
}
else
{
this.pattern = Pattern.compile(regex);
}
}
/* (non-Javadoc)
* @see java.util.Iterator#hasNext()
*/
@Override
public boolean hasNext()
{
boolean result;
preload();
if (this.nextLog == null)
{
result = false;
}
else
{
result = true;
}
//
return result;
}
/* (non-Javadoc)
* @see java.util.Iterator#next()
*/
@Override
public HttpAccessLog next()
{
HttpAccessLog result;
preload();
result = this.nextLog;
this.nextLog = null;
//
return result;
}
/**
* Forward.
*/
private void preload()
{
if (this.nextLog == null)
{
boolean ended = false;
while (!ended)
{
if (this.lineIterator.hasNext())
{
String line = this.lineIterator.next();
try
{
HttpAccessLog log = HttpAccessLogParser.parseLog(line, this.pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
this.nextLog = log;
ended = true;
}
}
catch (Exception exception)
{
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
exception.printStackTrace();
}
}
else
{
this.nextLog = null;
ended = true;
}
}
}
}
}

86
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogParser.java
View file

@ -1,86 +0,0 @@
/*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.http;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class HttpAccessLogParser.
*/
public class HttpAccessLogParser
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogParser.class);
// log_format combined '$remote_addr - $remote_user [$time_local] '
// '"$request" $status $body_bytes_sent '
// '"$http_referer" "$http_user_agent"';
public static final Pattern COMBINED_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>.*)\".*$");
/**
* Instantiates a new http access log parser.
*/
private HttpAccessLogParser()
{
}
/**
* Parses the log.
*
* @param line
* the line
* @return the http log
*/
public static HttpAccessLog parseLog(final String line, final Pattern pattern)
{
HttpAccessLog result;
Matcher matcher = pattern.matcher(line);
if (matcher.matches())
{
ZonedDateTime a = ZonedDateTime.now();
LocalDateTime b = a.toLocalDateTime();
result = new HttpAccessLog();
result.setIp(matcher.group("remoteAddress"));
result.setRemoteUser(matcher.group("remoteUser"));
result.setTime(ZonedDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
result.setRequest(matcher.group("request"));
result.setStatus(HttpStatusTable.instance().get(matcher.group("status")));
result.setBodyBytesSent(Long.valueOf(matcher.group("bodyBytesSent")));
result.setReferer(matcher.group("referer"));
result.setUserAgent(new UserAgent(matcher.group("userAgent")));
}
else
{
result = null;
}
//
return result;
}
}

86
src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java
View file

@ -1,86 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.http;
import java.io.IOException;
import java.util.Iterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class HttpAccessLogs.
*/
public class HttpAccessLogs implements Iterable<HttpAccessLog>
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogs.class);
private Files source;
private String regex;
/**
* Instantiates a new http access logs.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogs(final Files source) throws IOException
{
this(source, null);
}
/**
* Instantiates a new http log iterator.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public HttpAccessLogs(final Files source, final String regex) throws IOException
{
this.source = source;
this.regex = regex;
}
/* (non-Javadoc)
* @see java.lang.Iterable#iterator()
*/
@Override
public Iterator<HttpAccessLog> iterator()
{
HttpAccessLogIterator result;
try
{
result = new HttpAccessLogIterator(this.source, this.regex);
}
catch (IOException exception)
{
throw new IllegalArgumentException("Error with iterator.", exception);
}
//
return result;
}
}

4
src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -97,7 +97,7 @@ public class HttpLogIterator
try try
{ {
String line = this.iterator.next(); String line = this.iterator.next();
result = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN); result = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
} }
catch (Exception exception) catch (Exception exception)
{ {

7
src/fr/devinsy/statoolinfos/metrics/http/userAgentBotDetectorData.txt
View file

@ -1,6 +1,4 @@
AhrefsBot
^Apache-HttpClient/ ^Apache-HttpClient/
Barkrowler/
bot bot
BingPreview BingPreview
crawler crawler
@ -19,7 +17,6 @@ internal dummy connection
^Java/ ^Java/
^JGit/ ^JGit/
Let's Encrypt validation server; Let's Encrypt validation server;
ltx71
^MastoPeek ^MastoPeek
^mattermost- ^mattermost-
^Misskey/ ^Misskey/
@ -27,7 +24,6 @@ ltx71
monitoring monitoring
^newspaper/ ^newspaper/
^node-fetch/ ^node-fetch/
Nutch-
^okhttp/ ^okhttp/
^PeerTube/ ^PeerTube/
^PHP/ ^PHP/
@ -35,9 +31,6 @@ Nutch-
^python-requests/ ^python-requests/
^python/ ^python/
^Python/ ^Python/
^Python-urllib/
SemrushBot
^Synapse/ ^Synapse/
^Tusky/ ^Tusky/
YisouSpider YisouSpider
zgrab/

160
src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java
View file

@ -1,160 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.libreqr;
import java.io.File;
import java.io.IOException;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
/**
* The Class LibreQRProber.
*/
public class LibreQRProber
{
private static Logger logger = LoggerFactory.getLogger(LibreQRProber.class);
/**
* Instantiates a new privatebin prober.
*/
private LibreQRProber()
{
}
/**
* Probe.
*
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
{
PathCounters result;
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
// metrics.barcodes.count
// metrics.libreqr.barcodes.downloads
result = probeHttpAccessLog(httpLogs, httpLogRegex);
// metrics.service.files.bytes, metrics.libreqr.cache.bytes
// metrics.service.files.count, metrics.libreqr.cache.count
result.putAll(DatafilesProber.probe(dataPath));
//
return result;
}
/**
* Probe.
*
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
result = new PathCounters();
//
UserCounters users = new UserCounters();
UserCounters ipv4Users = new UserCounters();
UserCounters ipv6Users = new UserCounters();
Pattern USE_PATTERN = Pattern.compile("GET /temp/\\w+\\.png.*");
Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
//
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
{
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.barcodes.count
if ((log.getStatus().getCode() == 200) && (CREATE_PATTERN.matcher(log.getRequest()).matches()))
{
result.inc("metrics.barcodes.count", year, yearMonth, yearWeek, date);
}
// metrics.libreqr.barcodes.downloads
if (USE_PATTERN.matcher(log.getRequest()).matches())
{
result.inc("metrics.libreqr.barcodes.downloads", year, yearMonth, yearWeek, date);
}
}
//
result.putAll(users.getCounters("metrics.service.users"));
result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
}

91
src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java Normal file
View file

@ -0,0 +1,91 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.privatebin;
import java.io.File;
import java.io.IOException;
import java.time.LocalDate;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.TimeMark;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
* The Class PrivatebinDataAnalyzer.
*/
public class PrivatebinDataAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(PrivatebinDataAnalyzer.class);
/**
* Instantiates a new http access log prober.
*/
private PrivatebinDataAnalyzer()
{
}
/**
* Probe.
*
* @param dataDirectory
* the data directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
{
PathCounters result;
System.out.println("Probing directory [" + dataDirectory + "]");
result = new PathCounters();
if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
{
LocalDate now = LocalDate.now();
String year = TimeMark.yearOf(now).toString();
String yearMonth = TimeMark.yearMonthOf(now).toString();
String yearWeek = TimeMark.yearWeekOf(now).toString();
String date = TimeMark.dayOf(now).toString();
// metrics.service.files.bytes
long size = FileUtils.sizeOfDirectory(dataDirectory);
result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
// metrics.pastebins.count
long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date);
}
else
{
System.out.println("WARNING: Privatebin data path is not valid.");
}
//
return result;
}
}

215
src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java Normal file
View file

@ -0,0 +1,215 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.privatebin;
import java.io.File;
import java.io.IOException;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator;
/**
* The Class HttpAccessLogProber.
*/
public class PrivatebinHttpLogAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(PrivatebinHttpLogAnalyzer.class);
private PathCounters counters;
private UserCounters users;
private UserCounters ipv4Users;
private UserCounters ipv6Users;
/**
* Instantiates a new http access log prober.
*/
public PrivatebinHttpLogAnalyzer()
{
this.counters = new PathCounters();
this.users = new UserCounters();
this.ipv4Users = new UserCounters();
this.ipv6Users = new UserCounters();
}
/**
* Gets the counters.
*
* @return the counters
*/
public PathCounters getCounters()
{
PathCounters result;
result = new PathCounters();
result.putAll(this.counters);
result.putAll(this.users.getCounters("metrics.service.users"));
result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
/**
* Probe.
*
* @param file
* the file
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public void probe(final File file, final String patternRegex) throws IOException
{
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
//
Pattern pattern;
if (patternRegex == null)
{
pattern = HttpAccessLogAnalyzer.COMBINED_PATTERN;
}
else
{
pattern = Pattern.compile(patternRegex);
}
//
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
try
{
HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
probeLog(log);
}
}
catch (Exception exception)
{
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
exception.printStackTrace();
}
}
}
/**
* Probe log.
*
* @param log
* the log
*/
public void probeLog(final HttpAccessLog log)
{
if (log != null)
{
// logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString());
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
this.users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.pastebins.created
// metrics.pastebins.read
// metrics.pastebins.deleted
if (StringUtils.startsWith(log.getRequest(), "POST "))
{
this.counters.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
}
else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
{
if (StringUtils.contains(log.getRequest(), "&deletetoken="))
{
this.counters.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
}
else
{
this.counters.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
}
}
}
}
/**
* Probe.
*
* @param httpAccessLogs
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final String httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
PrivatebinHttpLogAnalyzer analyzer = new PrivatebinHttpLogAnalyzer();
for (File file : FilesUtils.searchByWildcard(httpAccessLogs))
{
analyzer.probe(file, httpRegex);
}
result = analyzer.getCounters();
//
return result;
}
}

94
src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java
View file

@ -21,18 +21,11 @@ package fr.devinsy.statoolinfos.metrics.privatebin;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters; import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.FilesUtils;
/** /**
* The Class PrivatebinProber. * The Class PrivatebinProber.
@ -44,80 +37,10 @@ public class PrivatebinProber
/** /**
* Instantiates a new privatebin prober. * Instantiates a new privatebin prober.
*/ */
private PrivatebinProber() public PrivatebinProber()
{ {
} }
private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
result = new PathCounters();
//
UserCounters users = new UserCounters();
UserCounters ipv4Users = new UserCounters();
UserCounters ipv6Users = new UserCounters();
//
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogs, httpRegex))
{
// logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString());
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.pastebins.created
// metrics.pastebins.read
// metrics.pastebins.deleted
if (StringUtils.startsWith(log.getRequest(), "POST "))
{
result.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
}
else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
{
if (StringUtils.contains(log.getRequest(), "&deletetoken="))
{
result.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
}
else
{
result.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
}
}
}
//
result.putAll(users.getCounters("metrics.service.users"));
result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
/** /**
* Probe. * Probe.
* *
@ -125,7 +48,7 @@ public class PrivatebinProber
* the http logs * the http logs
* @param httpLogRegex * @param httpLogRegex
* the http log regex * the http log regex
* @param datafileDirectory * @param dataPath
* the data path * the data path
* @return the path counters * @return the path counters
* @throws IOException * @throws IOException
@ -133,7 +56,7 @@ public class PrivatebinProber
* @throws StatoolInfosException * @throws StatoolInfosException
* the statool infos exception * the statool infos exception
*/ */
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File datafileDirectory) throws IOException, StatoolInfosException public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
{ {
PathCounters result; PathCounters result;
@ -143,18 +66,11 @@ public class PrivatebinProber
// metrics.pastebins.created // metrics.pastebins.created
// metrics.pastebins.read // metrics.pastebins.read
// metrics.pastebins.deleted // metrics.pastebins.deleted
result = probe(httpLogs, httpLogRegex); result = PrivatebinHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
// metrics.service.files.bytes // metrics.service.files.bytes
// metrics.service.files.count
result.putAll(DatafilesProber.probe(datafileDirectory));
// metrics.pastebins.count // metrics.pastebins.count
if ((datafileDirectory != null) && (datafileDirectory.exists()) && (datafileDirectory.isDirectory())) result.putAll(PrivatebinDataAnalyzer.probe(dataPath));
{
long count = FilesUtils.searchByWildcard(datafileDirectory.getAbsolutePath() + "/??/*").size();
result.set(count, "metrics.pastebins.count");
}
// metrics.pastebins.purged // metrics.pastebins.purged
// purged = count(n-1) - count(n) + created - delete // purged = count(n-1) - count(n) + created - delete

80
src/fr/devinsy/statoolinfos/metrics/util/DatabaseProber.java
View file

@ -1,80 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.util;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.DatabaseConfig;
import fr.devinsy.statoolinfos.metrics.PathCounters;
/**
* The Class DatabaseProber.
*/
public class DatabaseProber
{
private static Logger logger = LoggerFactory.getLogger(DatabaseProber.class);
/**
* Instantiates a new database prober.
*/
private DatabaseProber()
{
}
/**
* Probe.
*
* @param config
* the config
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static PathCounters probe(final DatabaseConfig config) throws IOException
{
PathCounters result;
result = new PathCounters();
if (config != null)
{
System.out.println("Probing directory [" + config.getName() + "]");
}
// if ((directory != null) && (directory.exists()) &&
// (directory.isDirectory()))
// {
// StringList timemarks = result.getNowTimeMarks();
//
// // metrics.service.database.bytes
// long size = FileUtils.sizeOfDirectory(directory);
// result.set(size, prefix + ".files.bytes", timemarks);
// }
// else
// {
// System.out.println("WARNING: datafile path not valid.");
// }
//
return result;
}
}

107
src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java
View file

@ -1,107 +0,0 @@
/*
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.util;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.strings.StringList;
/**
* The Class DatafilesProber.
*/
public class DatafilesProber
{
private static Logger logger = LoggerFactory.getLogger(DatafilesProber.class);
/**
* Instantiates a new datafiles prober.
*/
private DatafilesProber()
{
}
/**
* Probe.
*
* @param directory
* the directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File directory) throws IOException
{
PathCounters result;
result = probe(directory, "metrics.service");
//
return result;
}
/**
* Probe.
*
* @param directory
* the data directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File directory, final String prefix) throws IOException
{
PathCounters result;
System.out.println("Probing directory [" + directory + "]");
result = new PathCounters();
if ((directory != null) && (directory.exists()) && (directory.isDirectory()))
{
StringList timemarks = result.getNowTimeMarks();
// metrics.service.datafiles.bytes
long size = FileUtils.sizeOfDirectory(directory);
result.set(size, prefix + ".datafiles.bytes", timemarks);
// metrics.service.datafiles.count
long count = FilesUtils.listRecursively(directory.getAbsoluteFile()).size();
result.set(count, prefix + ".datafiles.count", timemarks);
}
else
{
System.out.println("WARNING: datafile path not valid.");
}
//
return result;
}
}

103
src/fr/devinsy/statoolinfos/util/Files.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -49,56 +49,6 @@ public class Files extends ArrayList<File>
super(initialCapacity); super(initialCapacity);
} }
/**
* Keep directories.
*
* @return the files
*/
public Files keepDirectoryType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isDirectory())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Keep file type.
*
* @return the files
*/
public Files keepFileType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isFile())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/** /**
* Keep. * Keep.
* *
@ -146,6 +96,7 @@ public class Files extends ArrayList<File>
while (iterator.hasNext()) while (iterator.hasNext())
{ {
File file = iterator.next(); File file = iterator.next();
if (!pattern.matcher(file.getAbsolutePath()).matches()) if (!pattern.matcher(file.getAbsolutePath()).matches())
{ {
iterator.remove(); iterator.remove();
@ -158,6 +109,56 @@ public class Files extends ArrayList<File>
return result; return result;
} }
/**
* Keep directories.
*
* @return the files
*/
public Files keepDirectoryType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isDirectory())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Keep file type.
*
* @return the files
*/
public Files keepFileType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isFile())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/** /**
* Removes the containing. * Removes the containing.
* *

181
src/fr/devinsy/statoolinfos/util/FilesLineIterator.java
View file

@ -1,181 +0,0 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.util;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class WildcardFileLineIterator.
*/
public class FilesLineIterator implements Iterator<String>
{
private static Logger logger = LoggerFactory.getLogger(FilesLineIterator.class);
private Iterator<File> fileIterator;
private LineIterator lineIterator;
private boolean print;
/**
* Instantiates a new wildcard file line iterator.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public FilesLineIterator(final Files source)
{
this.fileIterator = source.iterator();
this.lineIterator = null;
setPrintOn();
}
/**
* Forward.
*/
private void forward()
{
boolean ended = false;
while (!ended)
{
try
{
if (this.lineIterator == null)
{
if (this.fileIterator.hasNext())
{
File file = this.fileIterator.next();
if (this.print)
{
System.out.println("Iterating file [" + file.getAbsolutePath() + "]");
}
this.lineIterator = new LineIterator(file);
}
else
{
ended = true;
}
}
else if (this.lineIterator.hasNext())
{
ended = true;
}
else
{
this.lineIterator.close();
if (this.fileIterator.hasNext())
{
File file = this.fileIterator.next();
if (this.print)
{
System.out.println("Iterating file [" + file.getAbsolutePath() + "]");
}
this.lineIterator = new LineIterator(file);
}
else
{
this.lineIterator = null;
ended = true;
}
}
}
catch (IOException exception)
{
logger.error("ERROR READING.", exception);
if (this.lineIterator != null)
{
this.lineIterator.close();
this.lineIterator = null;
}
}
}
}
/* (non-Javadoc)
* @see java.util.Iterator#hasNext()
*/
@Override
public boolean hasNext()
{
boolean result;
forward();
if ((this.lineIterator == null) && (!this.fileIterator.hasNext()))
{
result = false;
}
else
{
result = true;
}
//
return result;
}
/* (non-Javadoc)
* @see java.util.Iterator#next()
*/
@Override
public String next()
{
String result;
try
{
forward();
if (this.lineIterator == null)
{
result = null;
}
else
{
result = this.lineIterator.next();
}
}
catch (IOException exception)
{
throw new IllegalArgumentException("ERROR READING.", exception);
}
//
return result;
}
/**
* Sets the print off.
*/
public void setPrintOff()
{
this.print = false;
}
public void setPrintOn()
{
this.print = true;
}
}

8
src/fr/devinsy/statoolinfos/util/FilesUtils.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -121,9 +121,7 @@ public class FilesUtils
{ {
// Get the first parent without wildcard. // Get the first parent without wildcard.
File parent = null; File parent = null;
File path = new File(source).getAbsoluteFile(); File current = new File(source);
File current = path;
boolean ended = false; boolean ended = false;
while (!ended) while (!ended)
{ {
@ -148,7 +146,7 @@ public class FilesUtils
// //
System.out.println("parent=" + parent); System.out.println("parent=" + parent);
String regex = path.toString().replace(".", "\\.").replace("?", ".").replace("*", ".*"); String regex = source.replace(".", "\\.").replace("?", ".").replace("*", ".*");
result = listRecursively(parent).keepPath(regex); result = listRecursively(parent).keepPath(regex);
} }