diff --git a/.classpath b/.classpath index 4cb3719..5f9bb89 100644 --- a/.classpath +++ b/.classpath @@ -28,5 +28,6 @@ + diff --git a/lib/json-simple-1.1.1-sources.jar b/lib/json-simple-1.1.1-sources.jar new file mode 100644 index 0000000..165e6f9 Binary files /dev/null and b/lib/json-simple-1.1.1-sources.jar differ diff --git a/lib/json-simple-1.1.1.jar b/lib/json-simple-1.1.1.jar new file mode 100644 index 0000000..dfd5856 Binary files /dev/null and b/lib/json-simple-1.1.1.jar differ diff --git a/src/fr/devinsy/statoolinfos/metrics/Prober.java b/src/fr/devinsy/statoolinfos/metrics/Prober.java index a5112b3..f8e3527 100644 --- a/src/fr/devinsy/statoolinfos/metrics/Prober.java +++ b/src/fr/devinsy/statoolinfos/metrics/Prober.java @@ -49,6 +49,7 @@ import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathProperty; import fr.devinsy.statoolinfos.properties.PathPropertyUtils; import fr.devinsy.statoolinfos.util.BuildInformation; +import fr.devinsy.statoolinfos.util.FilesUtils; import fr.devinsy.strings.StringList; import fr.devinsy.strings.StringsUtils; @@ -155,9 +156,9 @@ public class Prober String apiURL = configuration.get("conf.probe.gitea.api.url"); logger.info("apiURL=[{}]", apiURL); String apiToken = configuration.get("conf.probe.gitea.api.token"); - logger.info("apiToken=[{}]", apiURL); + logger.info("apiToken=[{}]", apiToken); - PathCounters metrics = GiteaProber.probe(httpLogs, null, apiURL, apiToken, dataDirectory); + PathCounters metrics = GiteaProber.probe(FilesUtils.searchByWildcard(httpLogs), null, apiURL, apiToken, dataDirectory); counters.putAll(metrics); } @@ -175,7 +176,7 @@ public class Prober File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles"); logger.info("dataDirectory=[{}]", dataDirectory); - PathCounters data = LibreQRProber.probe(source, httpAccessLogRegex, dataDirectory); + PathCounters data = LibreQRProber.probe(FilesUtils.searchByWildcard(source), httpAccessLogRegex, dataDirectory); counters.putAll(data); } @@ -212,7 +213,7 @@ public class Prober File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data"); logger.info("dataDirectory=[{}]", dataDirectory); - PathCounters data = PrivatebinProber.probe(httpAccessLogs, httpAccessLogRegex, dataDirectory); + PathCounters data = PrivatebinProber.probe(FilesUtils.searchByWildcard(httpAccessLogs), httpAccessLogRegex, dataDirectory); counters.putAll(data); } diff --git a/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaAPI.java b/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaAPI.java new file mode 100644 index 0000000..741387e --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaAPI.java @@ -0,0 +1,369 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.gitea; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.Charset; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class GiteaAPI. + */ +public class GiteaAPI +{ + private static Logger logger = LoggerFactory.getLogger(GiteaAPI.class); + + private String url; + private String token; + private JSONArray users; + private JSONArray organizations; + private JSONArray repositories; + + /** + * Instantiates a new gitea API. + */ + public GiteaAPI(final String url, final String token) throws IOException, ParseException + { + this.url = StringUtils.removeEnd(url, "/"); + this.token = token; + + // + String json = IOUtils.toString(new URL(url + "/api/v1/admin/users?limit=100000&token=" + token), Charset.defaultCharset()); + this.users = (JSONArray) (new JSONParser().parse(json)); + + // + json = IOUtils.toString(new URL(url + "/api/v1/admin/orgs?limit=100000&token=" + token), Charset.defaultCharset()); + this.organizations = (JSONArray) (new JSONParser().parse(json)); + + // + json = IOUtils.toString(new URL(url + "/api/v1/repos/search?limit=100000&token=" + token), Charset.defaultCharset()); + System.out.println(json); + this.repositories = (JSONArray) ((JSONObject) (new JSONParser().parse(json))).get("data"); + } + + /** + * Gets the active user count. + * + * @return the active user count + */ + public long getActiveUserCount() + { + long result; + + result = 0; + LocalDateTime limit = LocalDateTime.now().minusDays(31); + for (Object user : this.users) + { + String lastLogin = getJSONString(user, "last_login"); + + if (!StringUtils.isBlank(lastLogin)) + { + try + { + LocalDateTime date = LocalDateTime.parse(lastLogin, DateTimeFormatter.ISO_DATE_TIME); + if (date.isAfter(limit)) + { + result += 1; + } + } + catch (DateTimeParseException exception) + { + System.out.println("OUPS"); + } + } + } + + // + return result; + } + + /** + * Gets the fork repository count. + * + * @return the fork repository count + */ + public long getForkRepositoryCount() + { + long result; + + result = 0; + for (Object repository : this.repositories) + { + if (getJSONBoolean(repository, "fork")) + { + result += 1; + } + } + + // + return result; + } + + /** + * Gets the mirror repository count. + * + * @return the mirror repository count + */ + public long getMirrorRepositoryCount() + { + long result; + + result = 0; + for (Object repository : this.repositories) + { + if (getJSONBoolean(repository, "mirror")) + { + result += 1; + } + } + + // + return result; + } + + /** + * Gets the organization count. + * + * @return the organization count + */ + public long getOrganizationCount() + { + long result; + + result = this.organizations.size(); + + // + return result; + } + + /** + * Gets the private organization count. + * + * @return the private organization count + */ + public long getPrivateOrganizationCount() + { + long result; + + result = 0; + for (Object organization : this.organizations) + { + String visibility = getJSONString(organization, "visibility"); + + if (StringUtils.equals(visibility, "private")) + { + result += 1; + } + } + + // + return result; + } + + /** + * Gets the private repository count. + * + * @return the private repository count + */ + public long getPrivateRepositoryCount() + { + long result; + + result = 0; + for (Object repository : this.repositories) + { + if (getJSONBoolean(repository, "private")) + { + result += 1; + } + } + + // + return result; + } + + /** + * Gets the public organization count. + * + * @return the public organization count + */ + public long getPublicOrganizationCount() + { + long result; + + result = 0; + for (Object organization : this.organizations) + { + String visibility = getJSONString(organization, "visibility"); + + if (StringUtils.equals(visibility, "public")) + { + result += 1; + } + } + + // + return result; + } + + /** + * Gets the public repository count. + * + * @return the public repository count + */ + public long getPublicRepositoryCount() + { + long result; + + result = 0; + for (Object repository : this.repositories) + { + if (!getJSONBoolean(repository, "private")) + { + result += 1; + } + } + + // + return result; + } + + /** + * Gets the repository count. + * + * @return the repository count + */ + public long getRepositoryCount() + { + long result; + + result = this.repositories.size(); + + // + return result; + } + + /** + * Gets the user count. + * + * @return the user count + */ + public long getUserCount() + { + long result; + + result = this.users.size(); + + // + return result; + } + + /** + * Gets the boolean. + * + * @param object + * the object + * @param key + * the key + * @return the boolean + */ + private static boolean getJSONBoolean(final Object object, final String key) + { + Boolean result; + + Object value = ((JSONObject) object).get(key); + if (value == null) + { + result = false; + } + else + { + result = (Boolean) value; + } + + // + return result; + } + + /** + * Gets the long. + * + * @param object + * the object + * @param key + * the key + * @return the long + */ + public static long getJSONLong(final Object object, final String key) + { + long result; + + Object value = ((JSONObject) object).get(key); + if (value == null) + { + result = 0; + } + else + { + result = (Long) value; + } + + // + return result; + } + + /** + * Gets the as string. + * + * @param object + * the object + * @param key + * the key + * @return the as string + */ + public static String getJSONString(final Object object, final String key) + { + String result; + + Object value = ((JSONObject) object).get(key); + if (value == null) + { + result = ""; + } + else + { + result = (String) value; + } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java b/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java new file mode 100644 index 0000000..8eba63b --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java @@ -0,0 +1,300 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.gitea; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.StringUtils; +import org.json.simple.parser.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.metrics.PathCounters; +import fr.devinsy.statoolinfos.metrics.UserCounters; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; +import fr.devinsy.statoolinfos.metrics.util.DatabaseProber; +import fr.devinsy.statoolinfos.metrics.util.DatafilesProber; +import fr.devinsy.statoolinfos.util.Files; +import fr.devinsy.strings.StringList; + +/** + * The Class GiteaProber. + */ +public class GiteaProber +{ + private static Logger logger = LoggerFactory.getLogger(GiteaProber.class); + + /** + * Instantiates a new nextcloud prober. + */ + public GiteaProber() + { + } + + /** + * Probe. + * + * @param httpLogs + * the http logs + * @param httpLogRegex + * the http log regex + * @param dataPath + * the data path + * @return the path counters + * @throws IOException + * Signals that an I/O exception has occurred. + * @throws StatoolInfosException + * the statool infos exception + */ + public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final String apiURL, final String apiToken, final File dataPath) throws IOException, StatoolInfosException + { + PathCounters result; + + // metrics.service.users + // metrics.service.users.ipv4 + // metrics.service.users.ipv6 + result = probeHttpAccessLog(httpLogs, httpLogRegex); + + // metrics.service.database.bytes + result.putAll(DatabaseProber.probe(null)); + + // metrics.service.files.bytes + // metrics.service.files.count + result.putAll(DatafilesProber.probe(dataPath)); + + if (!StringUtils.isBlank(apiURL) && (!StringUtils.isBlank(apiToken))) + { + try + { + GiteaAPI gitea = new GiteaAPI("https://forge.devinsy.fr/", "6fd97c63c25cabf67ff90731ee79ab4568be89b0"); + + // metrics.service.accounts + // metrics.service.accounts.active + result.set(gitea.getUserCount(), "metrics.service.accounts"); + result.set(gitea.getActiveUserCount(), "metrics.service.accounts.active"); + + // metrics.forge.groups.* = + // metrics.forge.groups.private.* = + // metrics.forge.groups.public.* = + result.set(gitea.getOrganizationCount(), "metrics.forge.groups"); + result.set(gitea.getPublicOrganizationCount(), "metrics.forge.groups.public"); + result.set(gitea.getPrivateOrganizationCount(), "metrics.forge.groups.private"); + + // metrics.forge.projects.* = + // metrics.forge.repositories.* = + // metrics.forge.repositories.private.* = + // metrics.forge.repositories.public.* = + result.set(gitea.getRepositoryCount(), "metrics.forge.projects"); + result.set(gitea.getRepositoryCount(), "metrics.forge.repositories"); + result.set(gitea.getPublicRepositoryCount(), "metrics.forge.repositories.public"); + result.set(gitea.getPrivateRepositoryCount(), "metrics.forge.repositories.private"); + result.set(gitea.getForkRepositoryCount(), "metrics.forge.repositories.forks"); + result.set(gitea.getMirrorRepositoryCount(), "metrics.forge.repositories.mirrors"); + } + catch (IOException | ParseException exception) + { + exception.printStackTrace(); + } + } + + // /////////////////////////////////////// + + // # [Metrics spécifiques aux services de forges logicielles]. + // + // + // metrics.forge.commits.* = + // metrics.forge.mergerequests.* = + // metrics.forge.committers.* = + // metrics.forge.mergerequesters.* = + // metrics.forge.files.* = + + // + return result; + } + + /** + * Probe http access log. + * + * @param httpAccessLogFiles + * the http access log files + * @param httpRegex + * the http regex + * @return the path counters + * @throws IOException + * Signals that an I/O exception has occurred. + * @throws StatoolInfosException + * the statool infos exception + */ + private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException + { + PathCounters result; + + result = new PathCounters(); + + // metrics.service.users + // metrics.service.users.ipv4 + // metrics.service.users.ipv6 + UserCounters users = new UserCounters(); + UserCounters ipv4Users = new UserCounters(); + UserCounters ipv6Users = new UserCounters(); + + StringList regexList = new StringList(); + + /* + GET /admin + GET /admin/auths + GET /admin/config + GET /admin/emails + GET /admin/hooks + GET /admin/monitor + GET /admin/notices + GET /admin/orgs + GET /admin/repos + GET /admin/users + */ + regexList.add("GET /admin(/auths|/config|/emails|/hooks|/monitor|/notices|/orgs|/repos|/users)? .*"); + + /* + GET /cpm + GET /devinsy + */ + + /* + GET /devinsy/bacasable + GET /devinsy/bacasable/activity + GET /devinsy/bacasable/commits/branch/master + GET /devinsy/bacasable/compare/master...master + GET /devinsy/bacasable/graph + GET /devinsy/bacasable/issues + GET /devinsy/bacasable/issues/new?project=1 + GET /devinsy/bacasable/projects + GET /devinsy/bacasable/projects/1 + GET /devinsy/bacasable/projects/1/edit + GET /devinsy/bacasable/pulls + GET /devinsy/bacasable/pulls/1 + GET /devinsy/bacasable/releases + GET /devinsy/bacasable/releases/tag/0.5.0 + GET /devinsy/bacasable/tags + GET /devinsy/bacasable/wiki + GET /devinsy/bacasable/wiki/Home + */ + regexList.add("GET /\\S+/\\S+(/activity|/commits/branch/master|/compare/\\S+|/graph|/issues|/projects(/\\d+)|/pulls(/\\d+)|/releases(/tag/\\S+|/tags|/wiki|/wiki/\\S+)) .*"); + + /* + GET /devinsy/bacasable/settings + GET /devinsy/bacasable/settings/branches + GET /devinsy/bacasable/settings/collaboration + GET /devinsy/bacasable/settings/hooks + GET /devinsy/bacasable/settings/keys + GET /devinsy/bacasable/settings/lfs + GET /devinsy/bacasable/settings/lfs/locks + GET /devinsy/bacasable/settings/lfs/pointers + GET /devinsy/bacasable/settings/tags + */ + regexList.add("GET /\\S+/\\S+/settings(/branches|/collaboration|/hooks|/keys|/lfs|/lfs/locks|/lfs/pointers|/tags)? .*"); + + /* + GET /org/create + GET /org/devinsy/members + GET /org/devinsy/settings + GET /org/devinsy/teams + GET /org/devinsy/teams/new + */ + regexList.add("GET /org/(create|/\\S+/members|/\\S+/settings|/\\S+/teams|/\\S+/teams/new) .*"); + + /* + GET /explore/organizations + GET /explore/repos + GET /explore/users + */ + regexList.add("GET (/explore/organizations|/explore/repos|/explore/users) .*"); + + /* + GET /issues + GET /milestones + GET /pulls + */ + regexList.add("GET /(issues|milestones|pulls) .*"); + + /* + GET /repo/create + GET /repo/fork/7 + GET /repo/migrate + */ + regexList.add("GET /repo/(create|fork/\\d+|migrate) .*"); + + /* + GET /user/settings + GET /user/settings/account + GET /user/settings/appearance + GET /user/settings/applications + GET /user/settings/keys + GET /user/settings/organization + GET /user/settings/repos + GET /user/settings/security + */ + regexList.add("GET /user/settings(/account|/appearance|/applications|/keys|/organizations|/repos|/security)? .*"); + + String regex = regexList.toString("(", "|", ")"); + + Pattern USE_PATTERN = Pattern.compile(regex); + + // + for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex)) + { + // General HTTP access logs. + String year = log.getYear(); + String yearMonth = log.getYearMonth(); + String yearWeek = log.getYearWeek(); + String date = log.getDate(); + + // metrics.service.users + // metrics.service.users.ipv4 + // metrics.service.users.ipv6 + if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches())) + { + System.out.println(log.toStringLog()); + String key = String.format("%s---%s", log.getIp(), log.getUserAgent()); + + users.put(key, year, yearMonth, yearWeek, date); + + if (log.isIPv4()) + { + ipv4Users.put(key, year, yearMonth, yearWeek, date); + } + else + { + ipv6Users.put(key, year, yearMonth, yearWeek, date); + } + } + } + + // + result.putAll(users.getCounters("metrics.service.users")); + result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4")); + result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6")); + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java b/src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java index 0d8d954..76dc661 100644 --- a/src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java @@ -32,7 +32,6 @@ import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; import fr.devinsy.statoolinfos.metrics.util.DatafilesProber; import fr.devinsy.statoolinfos.util.Files; -import fr.devinsy.statoolinfos.util.FilesUtils; /** * The Class LibreQRProber. @@ -63,7 +62,7 @@ public class LibreQRProber * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException + public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException { PathCounters result; @@ -72,7 +71,7 @@ public class LibreQRProber // metrics.service.users.ipv6 // metrics.barcodes.count // metrics.libreqr.barcodes.downloads - result = probeHttpAccessLog(FilesUtils.searchByWildcard(httpLogs), httpLogRegex); + result = probeHttpAccessLog(httpLogs, httpLogRegex); // metrics.service.files.bytes, metrics.libreqr.cache.bytes // metrics.service.files.count, metrics.libreqr.cache.count diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java deleted file mode 100644 index 65710cb..0000000 --- a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (C) 2021 Christian Pierre MOMON - * - * This file is part of StatoolInfos, simple service statistics tool. - * - * StatoolInfos is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * StatoolInfos is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with StatoolInfos. If not, see . - */ -package fr.devinsy.statoolinfos.metrics.privatebin; - -import java.io.File; -import java.io.IOException; -import java.time.LocalDate; - -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import fr.devinsy.statoolinfos.core.StatoolInfosException; -import fr.devinsy.statoolinfos.metrics.PathCounters; -import fr.devinsy.statoolinfos.metrics.TimeMark; -import fr.devinsy.statoolinfos.util.FilesUtils; - -/** - * The Class PrivatebinDataAnalyzer. - */ -public class PrivatebinDataAnalyzer -{ - private static Logger logger = LoggerFactory.getLogger(PrivatebinDataAnalyzer.class); - - /** - * Instantiates a new http access log prober. - */ - private PrivatebinDataAnalyzer() - { - } - - /** - * Probe. - * - * @param dataDirectory - * the data directory - * @return the path counters - * @throws IOException - * Signals that an I/O exception has occurred. - * @throws StatoolInfosException - * the statool infos exception - */ - public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException - { - PathCounters result; - - System.out.println("Probing directory [" + dataDirectory + "]"); - - result = new PathCounters(); - - if ((dataDirectory.exists()) && (dataDirectory.isDirectory())) - { - LocalDate now = LocalDate.now(); - String year = TimeMark.yearOf(now).toString(); - String yearMonth = TimeMark.yearMonthOf(now).toString(); - String yearWeek = TimeMark.yearWeekOf(now).toString(); - String date = TimeMark.dayOf(now).toString(); - - // metrics.service.files.bytes - long size = FileUtils.sizeOfDirectory(dataDirectory); - result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date); - - // metrics.pastebins.count - long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size(); - result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date); - } - else - { - System.out.println("WARNING: Privatebin data path is not valid."); - } - - // - return result; - } -} diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java deleted file mode 100644 index 33e7df6..0000000 --- a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright (C) 2021-2022 Christian Pierre MOMON - * - * This file is part of StatoolInfos, simple service statistics tool. - * - * StatoolInfos is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * StatoolInfos is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with StatoolInfos. If not, see . - */ -package fr.devinsy.statoolinfos.metrics.privatebin; - -import java.io.File; -import java.io.IOException; -import java.util.regex.Pattern; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import fr.devinsy.statoolinfos.core.StatoolInfosException; -import fr.devinsy.statoolinfos.metrics.PathCounters; -import fr.devinsy.statoolinfos.metrics.UserCounters; -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; -import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogParser; -import fr.devinsy.statoolinfos.util.FilesUtils; -import fr.devinsy.statoolinfos.util.LineIterator; - -/** - * The Class HttpAccessLogProber. - */ -public class PrivatebinHttpLogAnalyzer -{ - private static Logger logger = LoggerFactory.getLogger(PrivatebinHttpLogAnalyzer.class); - - private PathCounters counters; - private UserCounters users; - private UserCounters ipv4Users; - private UserCounters ipv6Users; - - /** - * Instantiates a new http access log prober. - */ - public PrivatebinHttpLogAnalyzer() - { - this.counters = new PathCounters(); - this.users = new UserCounters(); - this.ipv4Users = new UserCounters(); - this.ipv6Users = new UserCounters(); - } - - /** - * Gets the counters. - * - * @return the counters - */ - public PathCounters getCounters() - { - PathCounters result; - - result = new PathCounters(); - result.putAll(this.counters); - - result.putAll(this.users.getCounters("metrics.service.users")); - result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4")); - result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6")); - - // - return result; - } - - /** - * Probe. - * - * @param file - * the file - * @throws IOException - * Signals that an I/O exception has occurred. - * @throws StatoolInfosException - * the statool infos exception - */ - public void probe(final File file, final String patternRegex) throws IOException - { - System.out.println("Probing file [" + file.getAbsolutePath() + "]"); - - // - Pattern pattern; - if (patternRegex == null) - { - pattern = HttpAccessLogParser.COMBINED_PATTERN; - } - else - { - pattern = Pattern.compile(patternRegex); - } - - // - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) - { - String line = iterator.next(); - - try - { - HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern); - if (log == null) - { - logger.warn("LINE IS NOT MATCHING [{}]", line); - } - else - { - probeLog(log); - } - } - catch (Exception exception) - { - logger.warn("Error parsing line [{}][{}]", line, exception.getMessage()); - exception.printStackTrace(); - } - } - } - - /** - * Probe log. - * - * @param log - * the log - */ - public void probeLog(final HttpAccessLog log) - { - if (log != null) - { - // logger.info("LINE IS MATCHING [{}]", log); - // logger.info(log.getHttpUserAgent().toString()); - - // General HTTP access logs. - String year = log.getYear(); - String yearMonth = log.getYearMonth(); - String yearWeek = log.getYearWeek(); - String date = log.getDate(); - - // metrics.service.users - // metrics.service.users.ipv4 - // metrics.service.users.ipv6 - if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid=")) - { - String key = String.format("%s---%s", log.getIp(), log.getUserAgent()); - this.users.put(key, year, yearMonth, yearWeek, date); - - if (log.isIPv4()) - { - this.ipv4Users.put(key, year, yearMonth, yearWeek, date); - } - else - { - this.ipv6Users.put(key, year, yearMonth, yearWeek, date); - } - } - - // metrics.pastebins.created - // metrics.pastebins.read - // metrics.pastebins.deleted - if (StringUtils.startsWith(log.getRequest(), "POST ")) - { - this.counters.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date); - } - else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid=")) - { - if (StringUtils.contains(log.getRequest(), "&deletetoken=")) - { - this.counters.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date); - } - else - { - this.counters.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date); - } - } - } - } - - /** - * Probe. - * - * @param httpAccessLogs - * the source - * @throws IOException - * Signals that an I/O exception has occurred. - * @throws StatoolInfosException - * the statool infos exception - */ - public static PathCounters probe(final String httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException - { - PathCounters result; - - PrivatebinHttpLogAnalyzer analyzer = new PrivatebinHttpLogAnalyzer(); - - for (File file : FilesUtils.searchByWildcard(httpAccessLogs)) - { - analyzer.probe(file, httpRegex); - } - - result = analyzer.getCounters(); - - // - return result; - } -} diff --git a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java index ff0c488..8629e74 100644 --- a/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java @@ -21,11 +21,18 @@ package fr.devinsy.statoolinfos.metrics.privatebin; import java.io.File; import java.io.IOException; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.metrics.PathCounters; +import fr.devinsy.statoolinfos.metrics.UserCounters; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs; +import fr.devinsy.statoolinfos.metrics.util.DatafilesProber; +import fr.devinsy.statoolinfos.util.Files; +import fr.devinsy.statoolinfos.util.FilesUtils; /** * The Class PrivatebinProber. @@ -37,10 +44,80 @@ public class PrivatebinProber /** * Instantiates a new privatebin prober. */ - public PrivatebinProber() + private PrivatebinProber() { } + private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException + { + PathCounters result; + + result = new PathCounters(); + + // + UserCounters users = new UserCounters(); + UserCounters ipv4Users = new UserCounters(); + UserCounters ipv6Users = new UserCounters(); + + // + for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogs, httpRegex)) + { + // logger.info("LINE IS MATCHING [{}]", log); + // logger.info(log.getHttpUserAgent().toString()); + + // General HTTP access logs. + String year = log.getYear(); + String yearMonth = log.getYearMonth(); + String yearWeek = log.getYearWeek(); + String date = log.getDate(); + + // metrics.service.users + // metrics.service.users.ipv4 + // metrics.service.users.ipv6 + if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid=")) + { + String key = String.format("%s---%s", log.getIp(), log.getUserAgent()); + users.put(key, year, yearMonth, yearWeek, date); + + if (log.isIPv4()) + { + ipv4Users.put(key, year, yearMonth, yearWeek, date); + } + else + { + ipv6Users.put(key, year, yearMonth, yearWeek, date); + } + } + + // metrics.pastebins.created + // metrics.pastebins.read + // metrics.pastebins.deleted + if (StringUtils.startsWith(log.getRequest(), "POST ")) + { + result.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date); + } + else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid=")) + { + if (StringUtils.contains(log.getRequest(), "&deletetoken=")) + { + result.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date); + } + else + { + result.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date); + } + } + } + + // + result.putAll(users.getCounters("metrics.service.users")); + result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4")); + result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6")); + + // + return result; + } + /** * Probe. * @@ -48,7 +125,7 @@ public class PrivatebinProber * the http logs * @param httpLogRegex * the http log regex - * @param dataPath + * @param datafileDirectory * the data path * @return the path counters * @throws IOException @@ -56,7 +133,7 @@ public class PrivatebinProber * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException + public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File datafileDirectory) throws IOException, StatoolInfosException { PathCounters result; @@ -66,11 +143,18 @@ public class PrivatebinProber // metrics.pastebins.created // metrics.pastebins.read // metrics.pastebins.deleted - result = PrivatebinHttpLogAnalyzer.probe(httpLogs, httpLogRegex); + result = probe(httpLogs, httpLogRegex); // metrics.service.files.bytes + // metrics.service.files.count + result.putAll(DatafilesProber.probe(datafileDirectory)); + // metrics.pastebins.count - result.putAll(PrivatebinDataAnalyzer.probe(dataPath)); + if ((datafileDirectory != null) && (datafileDirectory.exists()) && (datafileDirectory.isDirectory())) + { + long count = FilesUtils.searchByWildcard(datafileDirectory.getAbsolutePath() + "/??/*").size(); + result.set(count, "metrics.pastebins.count"); + } // metrics.pastebins.purged // purged = count(n-1) - count(n) + created - delete diff --git a/src/fr/devinsy/statoolinfos/metrics/util/DatabaseProber.java b/src/fr/devinsy/statoolinfos/metrics/util/DatabaseProber.java new file mode 100644 index 0000000..8997728 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/util/DatabaseProber.java @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.util; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.core.DatabaseConfig; +import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.metrics.PathCounters; + +/** + * The Class DatabaseProber. + */ +public class DatabaseProber +{ + private static Logger logger = LoggerFactory.getLogger(DatabaseProber.class); + + /** + * Instantiates a new database prober. + */ + private DatabaseProber() + { + } + + /** + * Probe. + * + * @param directory + * the directory + * @return the path counters + * @throws IOException + * Signals that an I/O exception has occurred. + * @throws StatoolInfosException + * the statool infos exception + */ + public static PathCounters probe(final DatabaseConfig config) throws IOException + { + PathCounters result; + + result = new PathCounters(); + + if (config != null) + { + System.out.println("Probing directory [" + config.getName() + "]"); + } + + // if ((directory != null) && (directory.exists()) && + // (directory.isDirectory())) + // { + // StringList timemarks = result.getNowTimeMarks(); + // + // // metrics.service.database.bytes + // long size = FileUtils.sizeOfDirectory(directory); + // result.set(size, prefix + ".files.bytes", timemarks); + // } + // else + // { + // System.out.println("WARNING: datafile path not valid."); + // } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java b/src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java index 8c95386..0d5d231 100644 --- a/src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java @@ -59,7 +59,7 @@ public class DatafilesProber { PathCounters result; - result = probe(directory, "metrics.files"); + result = probe(directory, "metrics.service"); // return result;