statool/statoolinfosweb
statool/statoolinfosweb
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Activity

Improved probing of Gitea, PrivateBin and LibreQR. Refactoring.

Browse source
This commit is contained in:
Christian P. MOMON 2022-02-15 01:45:39 +01:00
parent 0f4b1e2c2c
commit 5bd6e2c600
12 changed files with 850 additions and 319 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.classpath
View file

@ -28,5 +28,6 @@
<classpathentry kind="lib" path="lib/Logs/log4j-core-2.17.1.jar" sourcepath="lib/Logs/log4j-core-2.17.1-sources.jar"/>
<classpathentry kind="lib" path="lib/Logs/log4j-slf4j-impl-2.17.1.jar" sourcepath="lib/Logs/log4j-slf4j-impl-2.17.1-sources.jar"/>
<classpathentry kind="lib" path="lib/commons-io-2.11.0.jar" sourcepath="lib/commons-io-2.11.0-sources.jar"/>
<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar" sourcepath="lib/json-simple-1.1.1-sources.jar"/>
<classpathentry kind="output" path="bin"/>
</classpath>

BIN
lib/json-simple-1.1.1-sources.jar Normal file
View file

Binary file not shown.

BIN
lib/json-simple-1.1.1.jar Normal file
View file

Binary file not shown.

9
src/fr/devinsy/statoolinfos/metrics/Prober.java
View file

@ -49,6 +49,7 @@ import fr.devinsy.statoolinfos.properties.PathProperties;
import fr.devinsy.statoolinfos.properties.PathProperty;
import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
import fr.devinsy.statoolinfos.util.BuildInformation;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.strings.StringList;
import fr.devinsy.strings.StringsUtils;
@ -155,9 +156,9 @@ public class Prober
String apiURL = configuration.get("conf.probe.gitea.api.url");
logger.info("apiURL=[{}]", apiURL);
String apiToken = configuration.get("conf.probe.gitea.api.token");
logger.info("apiToken=[{}]", apiURL);
logger.info("apiToken=[{}]", apiToken);
PathCounters metrics = GiteaProber.probe(httpLogs, null, apiURL, apiToken, dataDirectory);
PathCounters metrics = GiteaProber.probe(FilesUtils.searchByWildcard(httpLogs), null, apiURL, apiToken, dataDirectory);
counters.putAll(metrics);
}
@ -175,7 +176,7 @@ public class Prober
File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles");
logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = LibreQRProber.probe(source, httpAccessLogRegex, dataDirectory);
PathCounters data = LibreQRProber.probe(FilesUtils.searchByWildcard(source), httpAccessLogRegex, dataDirectory);
counters.putAll(data);
}
@ -212,7 +213,7 @@ public class Prober
File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
logger.info("dataDirectory=[{}]", dataDirectory);
PathCounters data = PrivatebinProber.probe(httpAccessLogs, httpAccessLogRegex, dataDirectory);
PathCounters data = PrivatebinProber.probe(FilesUtils.searchByWildcard(httpAccessLogs), httpAccessLogRegex, dataDirectory);
counters.putAll(data);
}

369
src/fr/devinsy/statoolinfos/metrics/gitea/GiteaAPI.java Normal file
View file

@ -0,0 +1,369 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.gitea;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class GiteaAPI.
*/
public class GiteaAPI
{
private static Logger logger = LoggerFactory.getLogger(GiteaAPI.class);
private String url;
private String token;
private JSONArray users;
private JSONArray organizations;
private JSONArray repositories;
/**
* Instantiates a new gitea API.
*/
public GiteaAPI(final String url, final String token) throws IOException, ParseException
{
this.url = StringUtils.removeEnd(url, "/");
this.token = token;
//
String json = IOUtils.toString(new URL(url + "/api/v1/admin/users?limit=100000&token=" + token), Charset.defaultCharset());
this.users = (JSONArray) (new JSONParser().parse(json));
//
json = IOUtils.toString(new URL(url + "/api/v1/admin/orgs?limit=100000&token=" + token), Charset.defaultCharset());
this.organizations = (JSONArray) (new JSONParser().parse(json));
//
json = IOUtils.toString(new URL(url + "/api/v1/repos/search?limit=100000&token=" + token), Charset.defaultCharset());
System.out.println(json);
this.repositories = (JSONArray) ((JSONObject) (new JSONParser().parse(json))).get("data");
}
/**
* Gets the active user count.
*
* @return the active user count
*/
public long getActiveUserCount()
{
long result;
result = 0;
LocalDateTime limit = LocalDateTime.now().minusDays(31);
for (Object user : this.users)
{
String lastLogin = getJSONString(user, "last_login");
if (!StringUtils.isBlank(lastLogin))
{
try
{
LocalDateTime date = LocalDateTime.parse(lastLogin, DateTimeFormatter.ISO_DATE_TIME);
if (date.isAfter(limit))
{
result += 1;
}
}
catch (DateTimeParseException exception)
{
System.out.println("OUPS");
}
}
}
//
return result;
}
/**
* Gets the fork repository count.
*
* @return the fork repository count
*/
public long getForkRepositoryCount()
{
long result;
result = 0;
for (Object repository : this.repositories)
{
if (getJSONBoolean(repository, "fork"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the mirror repository count.
*
* @return the mirror repository count
*/
public long getMirrorRepositoryCount()
{
long result;
result = 0;
for (Object repository : this.repositories)
{
if (getJSONBoolean(repository, "mirror"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the organization count.
*
* @return the organization count
*/
public long getOrganizationCount()
{
long result;
result = this.organizations.size();
//
return result;
}
/**
* Gets the private organization count.
*
* @return the private organization count
*/
public long getPrivateOrganizationCount()
{
long result;
result = 0;
for (Object organization : this.organizations)
{
String visibility = getJSONString(organization, "visibility");
if (StringUtils.equals(visibility, "private"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the private repository count.
*
* @return the private repository count
*/
public long getPrivateRepositoryCount()
{
long result;
result = 0;
for (Object repository : this.repositories)
{
if (getJSONBoolean(repository, "private"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the public organization count.
*
* @return the public organization count
*/
public long getPublicOrganizationCount()
{
long result;
result = 0;
for (Object organization : this.organizations)
{
String visibility = getJSONString(organization, "visibility");
if (StringUtils.equals(visibility, "public"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the public repository count.
*
* @return the public repository count
*/
public long getPublicRepositoryCount()
{
long result;
result = 0;
for (Object repository : this.repositories)
{
if (!getJSONBoolean(repository, "private"))
{
result += 1;
}
}
//
return result;
}
/**
* Gets the repository count.
*
* @return the repository count
*/
public long getRepositoryCount()
{
long result;
result = this.repositories.size();
//
return result;
}
/**
* Gets the user count.
*
* @return the user count
*/
public long getUserCount()
{
long result;
result = this.users.size();
//
return result;
}
/**
* Gets the boolean.
*
* @param object
* the object
* @param key
* the key
* @return the boolean
*/
private static boolean getJSONBoolean(final Object object, final String key)
{
Boolean result;
Object value = ((JSONObject) object).get(key);
if (value == null)
{
result = false;
}
else
{
result = (Boolean) value;
}
//
return result;
}
/**
* Gets the long.
*
* @param object
* the object
* @param key
* the key
* @return the long
*/
public static long getJSONLong(final Object object, final String key)
{
long result;
Object value = ((JSONObject) object).get(key);
if (value == null)
{
result = 0;
}
else
{
result = (Long) value;
}
//
return result;
}
/**
* Gets the as string.
*
* @param object
* the object
* @param key
* the key
* @return the as string
*/
public static String getJSONString(final Object object, final String key)
{
String result;
Object value = ((JSONObject) object).get(key);
if (value == null)
{
result = "";
}
else
{
result = (String) value;
}
//
return result;
}
}

300
src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java Normal file
View file

@ -0,0 +1,300 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.gitea;
import java.io.File;
import java.io.IOException;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatabaseProber;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.strings.StringList;
/**
* The Class GiteaProber.
*/
public class GiteaProber
{
private static Logger logger = LoggerFactory.getLogger(GiteaProber.class);
/**
* Instantiates a new nextcloud prober.
*/
public GiteaProber()
{
}
/**
* Probe.
*
* @param httpLogs
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* the data path
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final String apiURL, final String apiToken, final File dataPath) throws IOException, StatoolInfosException
{
PathCounters result;
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
result = probeHttpAccessLog(httpLogs, httpLogRegex);
// metrics.service.database.bytes
result.putAll(DatabaseProber.probe(null));
// metrics.service.files.bytes
// metrics.service.files.count
result.putAll(DatafilesProber.probe(dataPath));
if (!StringUtils.isBlank(apiURL) && (!StringUtils.isBlank(apiToken)))
{
try
{
GiteaAPI gitea = new GiteaAPI("https://forge.devinsy.fr/", "6fd97c63c25cabf67ff90731ee79ab4568be89b0");
// metrics.service.accounts
// metrics.service.accounts.active
result.set(gitea.getUserCount(), "metrics.service.accounts");
result.set(gitea.getActiveUserCount(), "metrics.service.accounts.active");
// metrics.forge.groups.* =
// metrics.forge.groups.private.* =
// metrics.forge.groups.public.* =
result.set(gitea.getOrganizationCount(), "metrics.forge.groups");
result.set(gitea.getPublicOrganizationCount(), "metrics.forge.groups.public");
result.set(gitea.getPrivateOrganizationCount(), "metrics.forge.groups.private");
// metrics.forge.projects.* =
// metrics.forge.repositories.* =
// metrics.forge.repositories.private.* =
// metrics.forge.repositories.public.* =
result.set(gitea.getRepositoryCount(), "metrics.forge.projects");
result.set(gitea.getRepositoryCount(), "metrics.forge.repositories");
result.set(gitea.getPublicRepositoryCount(), "metrics.forge.repositories.public");
result.set(gitea.getPrivateRepositoryCount(), "metrics.forge.repositories.private");
result.set(gitea.getForkRepositoryCount(), "metrics.forge.repositories.forks");
result.set(gitea.getMirrorRepositoryCount(), "metrics.forge.repositories.mirrors");
}
catch (IOException | ParseException exception)
{
exception.printStackTrace();
}
}
// ///////////////////////////////////////
// # [Metrics spécifiques aux services de forges logicielles].
//
//
// metrics.forge.commits.* =
// metrics.forge.mergerequests.* =
// metrics.forge.committers.* =
// metrics.forge.mergerequesters.* =
// metrics.forge.files.* =
//
return result;
}
/**
* Probe http access log.
*
* @param httpAccessLogFiles
* the http access log files
* @param httpRegex
* the http regex
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
result = new PathCounters();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
UserCounters users = new UserCounters();
UserCounters ipv4Users = new UserCounters();
UserCounters ipv6Users = new UserCounters();
StringList regexList = new StringList();
/*
GET /admin
GET /admin/auths
GET /admin/config
GET /admin/emails
GET /admin/hooks
GET /admin/monitor
GET /admin/notices
GET /admin/orgs
GET /admin/repos
GET /admin/users
*/
regexList.add("GET /admin(/auths|/config|/emails|/hooks|/monitor|/notices|/orgs|/repos|/users)? .*");
/*
GET /cpm
GET /devinsy
*/
/*
GET /devinsy/bacasable
GET /devinsy/bacasable/activity
GET /devinsy/bacasable/commits/branch/master
GET /devinsy/bacasable/compare/master...master
GET /devinsy/bacasable/graph
GET /devinsy/bacasable/issues
GET /devinsy/bacasable/issues/new?project=1
GET /devinsy/bacasable/projects
GET /devinsy/bacasable/projects/1
GET /devinsy/bacasable/projects/1/edit
GET /devinsy/bacasable/pulls
GET /devinsy/bacasable/pulls/1
GET /devinsy/bacasable/releases
GET /devinsy/bacasable/releases/tag/0.5.0
GET /devinsy/bacasable/tags
GET /devinsy/bacasable/wiki
GET /devinsy/bacasable/wiki/Home
*/
regexList.add("GET /\\S+/\\S+(/activity|/commits/branch/master|/compare/\\S+|/graph|/issues|/projects(/\\d+)|/pulls(/\\d+)|/releases(/tag/\\S+|/tags|/wiki|/wiki/\\S+)) .*");
/*
GET /devinsy/bacasable/settings
GET /devinsy/bacasable/settings/branches
GET /devinsy/bacasable/settings/collaboration
GET /devinsy/bacasable/settings/hooks
GET /devinsy/bacasable/settings/keys
GET /devinsy/bacasable/settings/lfs
GET /devinsy/bacasable/settings/lfs/locks
GET /devinsy/bacasable/settings/lfs/pointers
GET /devinsy/bacasable/settings/tags
*/
regexList.add("GET /\\S+/\\S+/settings(/branches|/collaboration|/hooks|/keys|/lfs|/lfs/locks|/lfs/pointers|/tags)? .*");
/*
GET /org/create
GET /org/devinsy/members
GET /org/devinsy/settings
GET /org/devinsy/teams
GET /org/devinsy/teams/new
*/
regexList.add("GET /org/(create|/\\S+/members|/\\S+/settings|/\\S+/teams|/\\S+/teams/new) .*");
/*
GET /explore/organizations
GET /explore/repos
GET /explore/users
*/
regexList.add("GET (/explore/organizations|/explore/repos|/explore/users) .*");
/*
GET /issues
GET /milestones
GET /pulls
*/
regexList.add("GET /(issues|milestones|pulls) .*");
/*
GET /repo/create
GET /repo/fork/7
GET /repo/migrate
*/
regexList.add("GET /repo/(create|fork/\\d+|migrate) .*");
/*
GET /user/settings
GET /user/settings/account
GET /user/settings/appearance
GET /user/settings/applications
GET /user/settings/keys
GET /user/settings/organization
GET /user/settings/repos
GET /user/settings/security
*/
regexList.add("GET /user/settings(/account|/appearance|/applications|/keys|/organizations|/repos|/security)? .*");
String regex = regexList.toString("(", "|", ")");
Pattern USE_PATTERN = Pattern.compile(regex);
//
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
{
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
{
System.out.println(log.toStringLog());
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
}
//
result.putAll(users.getCounters("metrics.service.users"));
result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
}

5
src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java
View file

@ -32,7 +32,6 @@ import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
* The Class LibreQRProber.
@ -63,7 +62,7 @@ public class LibreQRProber
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
{
PathCounters result;
@ -72,7 +71,7 @@ public class LibreQRProber
// metrics.service.users.ipv6
// metrics.barcodes.count
// metrics.libreqr.barcodes.downloads
result = probeHttpAccessLog(FilesUtils.searchByWildcard(httpLogs), httpLogRegex);
result = probeHttpAccessLog(httpLogs, httpLogRegex);
// metrics.service.files.bytes, metrics.libreqr.cache.bytes
// metrics.service.files.count, metrics.libreqr.cache.count

91
src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java
View file

@ -1,91 +0,0 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.privatebin;
import java.io.File;
import java.io.IOException;
import java.time.LocalDate;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.TimeMark;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
* The Class PrivatebinDataAnalyzer.
*/
public class PrivatebinDataAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(PrivatebinDataAnalyzer.class);
/**
* Instantiates a new http access log prober.
*/
private PrivatebinDataAnalyzer()
{
}
/**
* Probe.
*
* @param dataDirectory
* the data directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
{
PathCounters result;
System.out.println("Probing directory [" + dataDirectory + "]");
result = new PathCounters();
if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
{
LocalDate now = LocalDate.now();
String year = TimeMark.yearOf(now).toString();
String yearMonth = TimeMark.yearMonthOf(now).toString();
String yearWeek = TimeMark.yearWeekOf(now).toString();
String date = TimeMark.dayOf(now).toString();
// metrics.service.files.bytes
long size = FileUtils.sizeOfDirectory(dataDirectory);
result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
// metrics.pastebins.count
long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date);
}
else
{
System.out.println("WARNING: Privatebin data path is not valid.");
}
//
return result;
}
}

215
src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java
View file

@ -1,215 +0,0 @@
/*
* Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.privatebin;
import java.io.File;
import java.io.IOException;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogParser;
import fr.devinsy.statoolinfos.util.FilesUtils;
import fr.devinsy.statoolinfos.util.LineIterator;
/**
* The Class HttpAccessLogProber.
*/
public class PrivatebinHttpLogAnalyzer
{
private static Logger logger = LoggerFactory.getLogger(PrivatebinHttpLogAnalyzer.class);
private PathCounters counters;
private UserCounters users;
private UserCounters ipv4Users;
private UserCounters ipv6Users;
/**
* Instantiates a new http access log prober.
*/
public PrivatebinHttpLogAnalyzer()
{
this.counters = new PathCounters();
this.users = new UserCounters();
this.ipv4Users = new UserCounters();
this.ipv6Users = new UserCounters();
}
/**
* Gets the counters.
*
* @return the counters
*/
public PathCounters getCounters()
{
PathCounters result;
result = new PathCounters();
result.putAll(this.counters);
result.putAll(this.users.getCounters("metrics.service.users"));
result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
/**
* Probe.
*
* @param file
* the file
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public void probe(final File file, final String patternRegex) throws IOException
{
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
//
Pattern pattern;
if (patternRegex == null)
{
pattern = HttpAccessLogParser.COMBINED_PATTERN;
}
else
{
pattern = Pattern.compile(patternRegex);
}
//
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
try
{
HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern);
if (log == null)
{
logger.warn("LINE IS NOT MATCHING [{}]", line);
}
else
{
probeLog(log);
}
}
catch (Exception exception)
{
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
exception.printStackTrace();
}
}
}
/**
* Probe log.
*
* @param log
* the log
*/
public void probeLog(final HttpAccessLog log)
{
if (log != null)
{
// logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString());
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
this.users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.pastebins.created
// metrics.pastebins.read
// metrics.pastebins.deleted
if (StringUtils.startsWith(log.getRequest(), "POST "))
{
this.counters.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
}
else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
{
if (StringUtils.contains(log.getRequest(), "&deletetoken="))
{
this.counters.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
}
else
{
this.counters.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
}
}
}
}
/**
* Probe.
*
* @param httpAccessLogs
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final String httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
PrivatebinHttpLogAnalyzer analyzer = new PrivatebinHttpLogAnalyzer();
for (File file : FilesUtils.searchByWildcard(httpAccessLogs))
{
analyzer.probe(file, httpRegex);
}
result = analyzer.getCounters();
//
return result;
}
}

94
src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java
View file

@ -21,11 +21,18 @@ package fr.devinsy.statoolinfos.metrics.privatebin;
import java.io.File;
import java.io.IOException;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
import fr.devinsy.statoolinfos.metrics.UserCounters;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
import fr.devinsy.statoolinfos.util.Files;
import fr.devinsy.statoolinfos.util.FilesUtils;
/**
* The Class PrivatebinProber.
@ -37,10 +44,80 @@ public class PrivatebinProber
/**
* Instantiates a new privatebin prober.
*/
public PrivatebinProber()
private PrivatebinProber()
{
}
private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
{
PathCounters result;
result = new PathCounters();
//
UserCounters users = new UserCounters();
UserCounters ipv4Users = new UserCounters();
UserCounters ipv6Users = new UserCounters();
//
for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogs, httpRegex))
{
// logger.info("LINE IS MATCHING [{}]", log);
// logger.info(log.getHttpUserAgent().toString());
// General HTTP access logs.
String year = log.getYear();
String yearMonth = log.getYearMonth();
String yearWeek = log.getYearWeek();
String date = log.getDate();
// metrics.service.users
// metrics.service.users.ipv4
// metrics.service.users.ipv6
if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
{
String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
users.put(key, year, yearMonth, yearWeek, date);
if (log.isIPv4())
{
ipv4Users.put(key, year, yearMonth, yearWeek, date);
}
else
{
ipv6Users.put(key, year, yearMonth, yearWeek, date);
}
}
// metrics.pastebins.created
// metrics.pastebins.read
// metrics.pastebins.deleted
if (StringUtils.startsWith(log.getRequest(), "POST "))
{
result.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
}
else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
{
if (StringUtils.contains(log.getRequest(), "&deletetoken="))
{
result.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
}
else
{
result.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
}
}
}
//
result.putAll(users.getCounters("metrics.service.users"));
result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
//
return result;
}
/**
* Probe.
*
@ -48,7 +125,7 @@ public class PrivatebinProber
* the http logs
* @param httpLogRegex
* the http log regex
* @param dataPath
* @param datafileDirectory
* the data path
* @return the path counters
* @throws IOException
@ -56,7 +133,7 @@ public class PrivatebinProber
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File datafileDirectory) throws IOException, StatoolInfosException
{
PathCounters result;
@ -66,11 +143,18 @@ public class PrivatebinProber
// metrics.pastebins.created
// metrics.pastebins.read
// metrics.pastebins.deleted
result = PrivatebinHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
result = probe(httpLogs, httpLogRegex);
// metrics.service.files.bytes
// metrics.service.files.count
result.putAll(DatafilesProber.probe(datafileDirectory));
// metrics.pastebins.count
result.putAll(PrivatebinDataAnalyzer.probe(dataPath));
if ((datafileDirectory != null) && (datafileDirectory.exists()) && (datafileDirectory.isDirectory()))
{
long count = FilesUtils.searchByWildcard(datafileDirectory.getAbsolutePath() + "/??/*").size();
result.set(count, "metrics.pastebins.count");
}
// metrics.pastebins.purged
// purged = count(n-1) - count(n) + created - delete

83
src/fr/devinsy/statoolinfos/metrics/util/DatabaseProber.java Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.metrics.util;
import java.io.IOException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.DatabaseConfig;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.metrics.PathCounters;
/**
* The Class DatabaseProber.
*/
public class DatabaseProber
{
private static Logger logger = LoggerFactory.getLogger(DatabaseProber.class);
/**
* Instantiates a new database prober.
*/
private DatabaseProber()
{
}
/**
* Probe.
*
* @param directory
* the directory
* @return the path counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static PathCounters probe(final DatabaseConfig config) throws IOException
{
PathCounters result;
result = new PathCounters();
if (config != null)
{
System.out.println("Probing directory [" + config.getName() + "]");
}
// if ((directory != null) && (directory.exists()) &&
// (directory.isDirectory()))
// {
// StringList timemarks = result.getNowTimeMarks();
//
// // metrics.service.database.bytes
// long size = FileUtils.sizeOfDirectory(directory);
// result.set(size, prefix + ".files.bytes", timemarks);
// }
// else
// {
// System.out.println("WARNING: datafile path not valid.");
// }
//
return result;
}
}

2
src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java
View file

@ -59,7 +59,7 @@ public class DatafilesProber
{
PathCounters result;
result = probe(directory, "metrics.files");
result = probe(directory, "metrics.service");
//
return result;