Improved probing of Gitea, PrivateBin and LibreQR. Refactoring.

.classpath

@@ -28,5 +28,6 @@
 	<classpathentry kind="lib" path="lib/Logs/log4j-core-2.17.1.jar" sourcepath="lib/Logs/log4j-core-2.17.1-sources.jar"/>
 	<classpathentry kind="lib" path="lib/Logs/log4j-slf4j-impl-2.17.1.jar" sourcepath="lib/Logs/log4j-slf4j-impl-2.17.1-sources.jar"/>
 	<classpathentry kind="lib" path="lib/commons-io-2.11.0.jar" sourcepath="lib/commons-io-2.11.0-sources.jar"/>
+	<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar" sourcepath="lib/json-simple-1.1.1-sources.jar"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

src/fr/devinsy/statoolinfos/metrics/Prober.java

@@ -49,6 +49,7 @@ import fr.devinsy.statoolinfos.properties.PathProperties;
 import fr.devinsy.statoolinfos.properties.PathProperty;
 import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
 import fr.devinsy.statoolinfos.util.BuildInformation;
+import fr.devinsy.statoolinfos.util.FilesUtils;
 import fr.devinsy.strings.StringList;
 import fr.devinsy.strings.StringsUtils;
 
@@ -155,9 +156,9 @@ public class Prober
                 String apiURL = configuration.get("conf.probe.gitea.api.url");
                 logger.info("apiURL=[{}]", apiURL);
                 String apiToken = configuration.get("conf.probe.gitea.api.token");
-                logger.info("apiToken=[{}]", apiURL);
+                logger.info("apiToken=[{}]", apiToken);
 
-                PathCounters metrics = GiteaProber.probe(httpLogs, null, apiURL, apiToken, dataDirectory);
+                PathCounters metrics = GiteaProber.probe(FilesUtils.searchByWildcard(httpLogs), null, apiURL, apiToken, dataDirectory);
                 counters.putAll(metrics);
             }
 
@@ -175,7 +176,7 @@ public class Prober
                 File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles");
                 logger.info("dataDirectory=[{}]", dataDirectory);
 
-                PathCounters data = LibreQRProber.probe(source, httpAccessLogRegex, dataDirectory);
+                PathCounters data = LibreQRProber.probe(FilesUtils.searchByWildcard(source), httpAccessLogRegex, dataDirectory);
                 counters.putAll(data);
             }
 
@@ -212,7 +213,7 @@ public class Prober
                 File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
                 logger.info("dataDirectory=[{}]", dataDirectory);
 
-                PathCounters data = PrivatebinProber.probe(httpAccessLogs, httpAccessLogRegex, dataDirectory);
+                PathCounters data = PrivatebinProber.probe(FilesUtils.searchByWildcard(httpAccessLogs), httpAccessLogRegex, dataDirectory);
                 counters.putAll(data);
             }
 

src/fr/devinsy/statoolinfos/metrics/gitea/GiteaAPI.java

@@ -0,0 +1,369 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.gitea;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The Class GiteaAPI.
+ */
+public class GiteaAPI
+{
+    private static Logger logger = LoggerFactory.getLogger(GiteaAPI.class);
+
+    private String url;
+    private String token;
+    private JSONArray users;
+    private JSONArray organizations;
+    private JSONArray repositories;
+
+    /**
+     * Instantiates a new gitea API.
+     */
+    public GiteaAPI(final String url, final String token) throws IOException, ParseException
+    {
+        this.url = StringUtils.removeEnd(url, "/");
+        this.token = token;
+
+        //
+        String json = IOUtils.toString(new URL(url + "/api/v1/admin/users?limit=100000&token=" + token), Charset.defaultCharset());
+        this.users = (JSONArray) (new JSONParser().parse(json));
+
+        //
+        json = IOUtils.toString(new URL(url + "/api/v1/admin/orgs?limit=100000&token=" + token), Charset.defaultCharset());
+        this.organizations = (JSONArray) (new JSONParser().parse(json));
+
+        //
+        json = IOUtils.toString(new URL(url + "/api/v1/repos/search?limit=100000&token=" + token), Charset.defaultCharset());
+        System.out.println(json);
+        this.repositories = (JSONArray) ((JSONObject) (new JSONParser().parse(json))).get("data");
+    }
+
+    /**
+     * Gets the active user count.
+     *
+     * @return the active user count
+     */
+    public long getActiveUserCount()
+    {
+        long result;
+
+        result = 0;
+        LocalDateTime limit = LocalDateTime.now().minusDays(31);
+        for (Object user : this.users)
+        {
+            String lastLogin = getJSONString(user, "last_login");
+
+            if (!StringUtils.isBlank(lastLogin))
+            {
+                try
+                {
+                    LocalDateTime date = LocalDateTime.parse(lastLogin, DateTimeFormatter.ISO_DATE_TIME);
+                    if (date.isAfter(limit))
+                    {
+                        result += 1;
+                    }
+                }
+                catch (DateTimeParseException exception)
+                {
+                    System.out.println("OUPS");
+                }
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the fork repository count.
+     *
+     * @return the fork repository count
+     */
+    public long getForkRepositoryCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object repository : this.repositories)
+        {
+            if (getJSONBoolean(repository, "fork"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the mirror repository count.
+     *
+     * @return the mirror repository count
+     */
+    public long getMirrorRepositoryCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object repository : this.repositories)
+        {
+            if (getJSONBoolean(repository, "mirror"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the organization count.
+     *
+     * @return the organization count
+     */
+    public long getOrganizationCount()
+    {
+        long result;
+
+        result = this.organizations.size();
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the private organization count.
+     *
+     * @return the private organization count
+     */
+    public long getPrivateOrganizationCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object organization : this.organizations)
+        {
+            String visibility = getJSONString(organization, "visibility");
+
+            if (StringUtils.equals(visibility, "private"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the private repository count.
+     *
+     * @return the private repository count
+     */
+    public long getPrivateRepositoryCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object repository : this.repositories)
+        {
+            if (getJSONBoolean(repository, "private"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the public organization count.
+     *
+     * @return the public organization count
+     */
+    public long getPublicOrganizationCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object organization : this.organizations)
+        {
+            String visibility = getJSONString(organization, "visibility");
+
+            if (StringUtils.equals(visibility, "public"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the public repository count.
+     *
+     * @return the public repository count
+     */
+    public long getPublicRepositoryCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object repository : this.repositories)
+        {
+            if (!getJSONBoolean(repository, "private"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the repository count.
+     *
+     * @return the repository count
+     */
+    public long getRepositoryCount()
+    {
+        long result;
+
+        result = this.repositories.size();
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the user count.
+     *
+     * @return the user count
+     */
+    public long getUserCount()
+    {
+        long result;
+
+        result = this.users.size();
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the boolean.
+     *
+     * @param object
+     *            the object
+     * @param key
+     *            the key
+     * @return the boolean
+     */
+    private static boolean getJSONBoolean(final Object object, final String key)
+    {
+        Boolean result;
+
+        Object value = ((JSONObject) object).get(key);
+        if (value == null)
+        {
+            result = false;
+        }
+        else
+        {
+            result = (Boolean) value;
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the long.
+     *
+     * @param object
+     *            the object
+     * @param key
+     *            the key
+     * @return the long
+     */
+    public static long getJSONLong(final Object object, final String key)
+    {
+        long result;
+
+        Object value = ((JSONObject) object).get(key);
+        if (value == null)
+        {
+            result = 0;
+        }
+        else
+        {
+            result = (Long) value;
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the as string.
+     *
+     * @param object
+     *            the object
+     * @param key
+     *            the key
+     * @return the as string
+     */
+    public static String getJSONString(final Object object, final String key)
+    {
+        String result;
+
+        Object value = ((JSONObject) object).get(key);
+        if (value == null)
+        {
+            result = "";
+        }
+        else
+        {
+            result = (String) value;
+        }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java

@@ -0,0 +1,300 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.gitea;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.StringUtils;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.metrics.UserCounters;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
+import fr.devinsy.statoolinfos.metrics.util.DatabaseProber;
+import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.strings.StringList;
+
+/**
+ * The Class GiteaProber.
+ */
+public class GiteaProber
+{
+    private static Logger logger = LoggerFactory.getLogger(GiteaProber.class);
+
+    /**
+     * Instantiates a new nextcloud prober.
+     */
+    public GiteaProber()
+    {
+    }
+
+    /**
+     * Probe.
+     *
+     * @param httpLogs
+     *            the http logs
+     * @param httpLogRegex
+     *            the http log regex
+     * @param dataPath
+     *            the data path
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final String apiURL, final String apiToken, final File dataPath) throws IOException, StatoolInfosException
+    {
+        PathCounters result;
+
+        // metrics.service.users
+        // metrics.service.users.ipv4
+        // metrics.service.users.ipv6
+        result = probeHttpAccessLog(httpLogs, httpLogRegex);
+
+        // metrics.service.database.bytes
+        result.putAll(DatabaseProber.probe(null));
+
+        // metrics.service.files.bytes
+        // metrics.service.files.count
+        result.putAll(DatafilesProber.probe(dataPath));
+
+        if (!StringUtils.isBlank(apiURL) && (!StringUtils.isBlank(apiToken)))
+        {
+            try
+            {
+                GiteaAPI gitea = new GiteaAPI("https://forge.devinsy.fr/", "6fd97c63c25cabf67ff90731ee79ab4568be89b0");
+
+                // metrics.service.accounts
+                // metrics.service.accounts.active
+                result.set(gitea.getUserCount(), "metrics.service.accounts");
+                result.set(gitea.getActiveUserCount(), "metrics.service.accounts.active");
+
+                // metrics.forge.groups.* =
+                // metrics.forge.groups.private.* =
+                // metrics.forge.groups.public.* =
+                result.set(gitea.getOrganizationCount(), "metrics.forge.groups");
+                result.set(gitea.getPublicOrganizationCount(), "metrics.forge.groups.public");
+                result.set(gitea.getPrivateOrganizationCount(), "metrics.forge.groups.private");
+
+                // metrics.forge.projects.* =
+                // metrics.forge.repositories.* =
+                // metrics.forge.repositories.private.* =
+                // metrics.forge.repositories.public.* =
+                result.set(gitea.getRepositoryCount(), "metrics.forge.projects");
+                result.set(gitea.getRepositoryCount(), "metrics.forge.repositories");
+                result.set(gitea.getPublicRepositoryCount(), "metrics.forge.repositories.public");
+                result.set(gitea.getPrivateRepositoryCount(), "metrics.forge.repositories.private");
+                result.set(gitea.getForkRepositoryCount(), "metrics.forge.repositories.forks");
+                result.set(gitea.getMirrorRepositoryCount(), "metrics.forge.repositories.mirrors");
+            }
+            catch (IOException | ParseException exception)
+            {
+                exception.printStackTrace();
+            }
+        }
+
+        // ///////////////////////////////////////
+
+        // # [Metrics spécifiques aux services de forges logicielles].
+        //
+        //
+        // metrics.forge.commits.* =
+        // metrics.forge.mergerequests.* =
+        // metrics.forge.committers.* =
+        // metrics.forge.mergerequesters.* =
+        // metrics.forge.files.* =
+
+        //
+        return result;
+    }
+
+    /**
+     * Probe http access log.
+     *
+     * @param httpAccessLogFiles
+     *            the http access log files
+     * @param httpRegex
+     *            the http regex
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
+    {
+        PathCounters result;
+
+        result = new PathCounters();
+
+        // metrics.service.users
+        // metrics.service.users.ipv4
+        // metrics.service.users.ipv6
+        UserCounters users = new UserCounters();
+        UserCounters ipv4Users = new UserCounters();
+        UserCounters ipv6Users = new UserCounters();
+
+        StringList regexList = new StringList();
+
+        /*
+        GET /admin
+        GET /admin/auths
+        GET /admin/config
+        GET /admin/emails
+        GET /admin/hooks
+        GET /admin/monitor
+        GET /admin/notices
+        GET /admin/orgs
+        GET /admin/repos
+        GET /admin/users
+        */
+        regexList.add("GET /admin(/auths|/config|/emails|/hooks|/monitor|/notices|/orgs|/repos|/users)? .*");
+
+        /*
+        GET /cpm
+        GET /devinsy
+        */
+
+        /*
+        GET /devinsy/bacasable
+        GET /devinsy/bacasable/activity
+        GET /devinsy/bacasable/commits/branch/master
+        GET /devinsy/bacasable/compare/master...master
+        GET /devinsy/bacasable/graph
+        GET /devinsy/bacasable/issues
+        GET /devinsy/bacasable/issues/new?project=1
+        GET /devinsy/bacasable/projects
+        GET /devinsy/bacasable/projects/1
+        GET /devinsy/bacasable/projects/1/edit
+        GET /devinsy/bacasable/pulls
+        GET /devinsy/bacasable/pulls/1
+        GET /devinsy/bacasable/releases
+        GET /devinsy/bacasable/releases/tag/0.5.0
+        GET /devinsy/bacasable/tags
+        GET /devinsy/bacasable/wiki
+        GET /devinsy/bacasable/wiki/Home
+        */
+        regexList.add("GET /\\S+/\\S+(/activity|/commits/branch/master|/compare/\\S+|/graph|/issues|/projects(/\\d+)|/pulls(/\\d+)|/releases(/tag/\\S+|/tags|/wiki|/wiki/\\S+)) .*");
+
+        /*
+        GET /devinsy/bacasable/settings
+        GET /devinsy/bacasable/settings/branches
+        GET /devinsy/bacasable/settings/collaboration
+        GET /devinsy/bacasable/settings/hooks
+        GET /devinsy/bacasable/settings/keys
+        GET /devinsy/bacasable/settings/lfs
+        GET /devinsy/bacasable/settings/lfs/locks
+        GET /devinsy/bacasable/settings/lfs/pointers
+        GET /devinsy/bacasable/settings/tags
+         */
+        regexList.add("GET /\\S+/\\S+/settings(/branches|/collaboration|/hooks|/keys|/lfs|/lfs/locks|/lfs/pointers|/tags)? .*");
+
+        /*
+        GET /org/create
+        GET /org/devinsy/members
+        GET /org/devinsy/settings
+        GET /org/devinsy/teams
+        GET /org/devinsy/teams/new
+         */
+        regexList.add("GET /org/(create|/\\S+/members|/\\S+/settings|/\\S+/teams|/\\S+/teams/new) .*");
+
+        /*
+        GET /explore/organizations
+        GET /explore/repos
+        GET /explore/users
+        */
+        regexList.add("GET (/explore/organizations|/explore/repos|/explore/users) .*");
+
+        /*
+        GET /issues
+        GET /milestones
+        GET /pulls
+         */
+        regexList.add("GET /(issues|milestones|pulls) .*");
+
+        /*
+        GET /repo/create
+        GET /repo/fork/7
+        GET /repo/migrate
+        */
+        regexList.add("GET /repo/(create|fork/\\d+|migrate) .*");
+
+        /*
+        GET /user/settings
+        GET /user/settings/account
+        GET /user/settings/appearance
+        GET /user/settings/applications
+        GET /user/settings/keys
+        GET /user/settings/organization
+        GET /user/settings/repos
+        GET /user/settings/security
+         */
+        regexList.add("GET /user/settings(/account|/appearance|/applications|/keys|/organizations|/repos|/security)? .*");
+
+        String regex = regexList.toString("(", "|", ")");
+
+        Pattern USE_PATTERN = Pattern.compile(regex);
+
+        //
+        for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
+        {
+            // General HTTP access logs.
+            String year = log.getYear();
+            String yearMonth = log.getYearMonth();
+            String yearWeek = log.getYearWeek();
+            String date = log.getDate();
+
+            // metrics.service.users
+            // metrics.service.users.ipv4
+            // metrics.service.users.ipv6
+            if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
+            {
+                System.out.println(log.toStringLog());
+                String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
+
+                users.put(key, year, yearMonth, yearWeek, date);
+
+                if (log.isIPv4())
+                {
+                    ipv4Users.put(key, year, yearMonth, yearWeek, date);
+                }
+                else
+                {
+                    ipv6Users.put(key, year, yearMonth, yearWeek, date);
+                }
+            }
+        }
+
+        //
+        result.putAll(users.getCounters("metrics.service.users"));
+        result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
+        result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java

@@ -32,7 +32,6 @@ import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
 import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
 import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
 import fr.devinsy.statoolinfos.util.Files;
-import fr.devinsy.statoolinfos.util.FilesUtils;
 
 /**
  * The Class LibreQRProber.
@@ -63,7 +62,7 @@ public class LibreQRProber
      * @throws StatoolInfosException
      *             the statool infos exception
      */
-    public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
+    public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
     {
         PathCounters result;
 
@@ -72,7 +71,7 @@ public class LibreQRProber
         // metrics.service.users.ipv6
         // metrics.barcodes.count
         // metrics.libreqr.barcodes.downloads
-        result = probeHttpAccessLog(FilesUtils.searchByWildcard(httpLogs), httpLogRegex);
+        result = probeHttpAccessLog(httpLogs, httpLogRegex);
 
         // metrics.service.files.bytes, metrics.libreqr.cache.bytes
         // metrics.service.files.count, metrics.libreqr.cache.count

src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java

@@ -1,91 +0,0 @@
-/*
- * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
- * 
- * This file is part of StatoolInfos, simple service statistics tool.
- * 
- * StatoolInfos is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- * 
- * StatoolInfos is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- * 
- * You should have received a copy of the GNU Affero General Public License
- * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
- */
-package fr.devinsy.statoolinfos.metrics.privatebin;
-
-import java.io.File;
-import java.io.IOException;
-import java.time.LocalDate;
-
-import org.apache.commons.io.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import fr.devinsy.statoolinfos.core.StatoolInfosException;
-import fr.devinsy.statoolinfos.metrics.PathCounters;
-import fr.devinsy.statoolinfos.metrics.TimeMark;
-import fr.devinsy.statoolinfos.util.FilesUtils;
-
-/**
- * The Class PrivatebinDataAnalyzer.
- */
-public class PrivatebinDataAnalyzer
-{
-    private static Logger logger = LoggerFactory.getLogger(PrivatebinDataAnalyzer.class);
-
-    /**
-     * Instantiates a new http access log prober.
-     */
-    private PrivatebinDataAnalyzer()
-    {
-    }
-
-    /**
-     * Probe.
-     *
-     * @param dataDirectory
-     *            the data directory
-     * @return the path counters
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
-     * @throws StatoolInfosException
-     *             the statool infos exception
-     */
-    public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
-    {
-        PathCounters result;
-
-        System.out.println("Probing directory [" + dataDirectory + "]");
-
-        result = new PathCounters();
-
-        if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
-        {
-            LocalDate now = LocalDate.now();
-            String year = TimeMark.yearOf(now).toString();
-            String yearMonth = TimeMark.yearMonthOf(now).toString();
-            String yearWeek = TimeMark.yearWeekOf(now).toString();
-            String date = TimeMark.dayOf(now).toString();
-
-            // metrics.service.files.bytes
-            long size = FileUtils.sizeOfDirectory(dataDirectory);
-            result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
-
-            // metrics.pastebins.count
-            long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
-            result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date);
-        }
-        else
-        {
-            System.out.println("WARNING: Privatebin data path is not valid.");
-        }
-
-        //
-        return result;
-    }
-}

src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java

@@ -1,215 +0,0 @@
-/*
- * Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
- * 
- * This file is part of StatoolInfos, simple service statistics tool.
- * 
- * StatoolInfos is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- * 
- * StatoolInfos is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- * 
- * You should have received a copy of the GNU Affero General Public License
- * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
- */
-package fr.devinsy.statoolinfos.metrics.privatebin;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import fr.devinsy.statoolinfos.core.StatoolInfosException;
-import fr.devinsy.statoolinfos.metrics.PathCounters;
-import fr.devinsy.statoolinfos.metrics.UserCounters;
-import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
-import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogParser;
-import fr.devinsy.statoolinfos.util.FilesUtils;
-import fr.devinsy.statoolinfos.util.LineIterator;
-
-/**
- * The Class HttpAccessLogProber.
- */
-public class PrivatebinHttpLogAnalyzer
-{
-    private static Logger logger = LoggerFactory.getLogger(PrivatebinHttpLogAnalyzer.class);
-
-    private PathCounters counters;
-    private UserCounters users;
-    private UserCounters ipv4Users;
-    private UserCounters ipv6Users;
-
-    /**
-     * Instantiates a new http access log prober.
-     */
-    public PrivatebinHttpLogAnalyzer()
-    {
-        this.counters = new PathCounters();
-        this.users = new UserCounters();
-        this.ipv4Users = new UserCounters();
-        this.ipv6Users = new UserCounters();
-    }
-
-    /**
-     * Gets the counters.
-     *
-     * @return the counters
-     */
-    public PathCounters getCounters()
-    {
-        PathCounters result;
-
-        result = new PathCounters();
-        result.putAll(this.counters);
-
-        result.putAll(this.users.getCounters("metrics.service.users"));
-        result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
-        result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
-
-        //
-        return result;
-    }
-
-    /**
-     * Probe.
-     *
-     * @param file
-     *            the file
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
-     * @throws StatoolInfosException
-     *             the statool infos exception
-     */
-    public void probe(final File file, final String patternRegex) throws IOException
-    {
-        System.out.println("Probing file [" + file.getAbsolutePath() + "]");
-
-        //
-        Pattern pattern;
-        if (patternRegex == null)
-        {
-            pattern = HttpAccessLogParser.COMBINED_PATTERN;
-        }
-        else
-        {
-            pattern = Pattern.compile(patternRegex);
-        }
-
-        //
-        LineIterator iterator = new LineIterator(file);
-        while (iterator.hasNext())
-        {
-            String line = iterator.next();
-
-            try
-            {
-                HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern);
-                if (log == null)
-                {
-                    logger.warn("LINE IS NOT MATCHING [{}]", line);
-                }
-                else
-                {
-                    probeLog(log);
-                }
-            }
-            catch (Exception exception)
-            {
-                logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
-                exception.printStackTrace();
-            }
-        }
-    }
-
-    /**
-     * Probe log.
-     *
-     * @param log
-     *            the log
-     */
-    public void probeLog(final HttpAccessLog log)
-    {
-        if (log != null)
-        {
-            // logger.info("LINE IS MATCHING [{}]", log);
-            // logger.info(log.getHttpUserAgent().toString());
-
-            // General HTTP access logs.
-            String year = log.getYear();
-            String yearMonth = log.getYearMonth();
-            String yearWeek = log.getYearWeek();
-            String date = log.getDate();
-
-            // metrics.service.users
-            // metrics.service.users.ipv4
-            // metrics.service.users.ipv6
-            if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
-            {
-                String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
-                this.users.put(key, year, yearMonth, yearWeek, date);
-
-                if (log.isIPv4())
-                {
-                    this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
-                }
-                else
-                {
-                    this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
-                }
-            }
-
-            // metrics.pastebins.created
-            // metrics.pastebins.read
-            // metrics.pastebins.deleted
-            if (StringUtils.startsWith(log.getRequest(), "POST "))
-            {
-                this.counters.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
-            }
-            else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
-            {
-                if (StringUtils.contains(log.getRequest(), "&deletetoken="))
-                {
-                    this.counters.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
-                }
-                else
-                {
-                    this.counters.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
-                }
-            }
-        }
-    }
-
-    /**
-     * Probe.
-     *
-     * @param httpAccessLogs
-     *            the source
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
-     * @throws StatoolInfosException
-     *             the statool infos exception
-     */
-    public static PathCounters probe(final String httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
-    {
-        PathCounters result;
-
-        PrivatebinHttpLogAnalyzer analyzer = new PrivatebinHttpLogAnalyzer();
-
-        for (File file : FilesUtils.searchByWildcard(httpAccessLogs))
-        {
-            analyzer.probe(file, httpRegex);
-        }
-
-        result = analyzer.getCounters();
-
-        //
-        return result;
-    }
-}

src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java

@@ -21,11 +21,18 @@ package fr.devinsy.statoolinfos.metrics.privatebin;
 import java.io.File;
 import java.io.IOException;
 
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import fr.devinsy.statoolinfos.core.StatoolInfosException;
 import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.metrics.UserCounters;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
+import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.statoolinfos.util.FilesUtils;
 
 /**
  * The Class PrivatebinProber.
@@ -37,10 +44,80 @@ public class PrivatebinProber
     /**
      * Instantiates a new privatebin prober.
      */
-    public PrivatebinProber()
+    private PrivatebinProber()
     {
     }
 
+    private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
+    {
+        PathCounters result;
+
+        result = new PathCounters();
+
+        //
+        UserCounters users = new UserCounters();
+        UserCounters ipv4Users = new UserCounters();
+        UserCounters ipv6Users = new UserCounters();
+
+        //
+        for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogs, httpRegex))
+        {
+            // logger.info("LINE IS MATCHING [{}]", log);
+            // logger.info(log.getHttpUserAgent().toString());
+
+            // General HTTP access logs.
+            String year = log.getYear();
+            String yearMonth = log.getYearMonth();
+            String yearWeek = log.getYearWeek();
+            String date = log.getDate();
+
+            // metrics.service.users
+            // metrics.service.users.ipv4
+            // metrics.service.users.ipv6
+            if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
+            {
+                String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
+                users.put(key, year, yearMonth, yearWeek, date);
+
+                if (log.isIPv4())
+                {
+                    ipv4Users.put(key, year, yearMonth, yearWeek, date);
+                }
+                else
+                {
+                    ipv6Users.put(key, year, yearMonth, yearWeek, date);
+                }
+            }
+
+            // metrics.pastebins.created
+            // metrics.pastebins.read
+            // metrics.pastebins.deleted
+            if (StringUtils.startsWith(log.getRequest(), "POST "))
+            {
+                result.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
+            }
+            else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
+            {
+                if (StringUtils.contains(log.getRequest(), "&deletetoken="))
+                {
+                    result.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
+                }
+                else
+                {
+                    result.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
+                }
+            }
+        }
+
+        //
+        result.putAll(users.getCounters("metrics.service.users"));
+        result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
+        result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
+
+        //
+        return result;
+    }
+
     /**
      * Probe.
      *
@@ -48,7 +125,7 @@ public class PrivatebinProber
      *            the http logs
      * @param httpLogRegex
      *            the http log regex
-     * @param dataPath
+     * @param datafileDirectory
      *            the data path
      * @return the path counters
      * @throws IOException
@@ -56,7 +133,7 @@ public class PrivatebinProber
      * @throws StatoolInfosException
      *             the statool infos exception
      */
-    public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
+    public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File datafileDirectory) throws IOException, StatoolInfosException
     {
         PathCounters result;
 
@@ -66,11 +143,18 @@ public class PrivatebinProber
         // metrics.pastebins.created
         // metrics.pastebins.read
         // metrics.pastebins.deleted
-        result = PrivatebinHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
+        result = probe(httpLogs, httpLogRegex);
 
         // metrics.service.files.bytes
+        // metrics.service.files.count
+        result.putAll(DatafilesProber.probe(datafileDirectory));
+
         // metrics.pastebins.count
-        result.putAll(PrivatebinDataAnalyzer.probe(dataPath));
+        if ((datafileDirectory != null) && (datafileDirectory.exists()) && (datafileDirectory.isDirectory()))
+        {
+            long count = FilesUtils.searchByWildcard(datafileDirectory.getAbsolutePath() + "/??/*").size();
+            result.set(count, "metrics.pastebins.count");
+        }
 
         // metrics.pastebins.purged
         // purged = count(n-1) - count(n) + created - delete

src/fr/devinsy/statoolinfos/metrics/util/DatabaseProber.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.util;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.core.DatabaseConfig;
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+import fr.devinsy.statoolinfos.metrics.PathCounters;
+
+/**
+ * The Class DatabaseProber.
+ */
+public class DatabaseProber
+{
+    private static Logger logger = LoggerFactory.getLogger(DatabaseProber.class);
+
+    /**
+     * Instantiates a new database prober.
+     */
+    private DatabaseProber()
+    {
+    }
+
+    /**
+     * Probe.
+     *
+     * @param directory
+     *            the directory
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    public static PathCounters probe(final DatabaseConfig config) throws IOException
+    {
+        PathCounters result;
+
+        result = new PathCounters();
+
+        if (config != null)
+        {
+            System.out.println("Probing directory [" + config.getName() + "]");
+        }
+
+        // if ((directory != null) && (directory.exists()) &&
+        // (directory.isDirectory()))
+        // {
+        // StringList timemarks = result.getNowTimeMarks();
+        //
+        // // metrics.service.database.bytes
+        // long size = FileUtils.sizeOfDirectory(directory);
+        // result.set(size, prefix + ".files.bytes", timemarks);
+        // }
+        // else
+        // {
+        // System.out.println("WARNING: datafile path not valid.");
+        // }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java

@@ -59,7 +59,7 @@ public class DatafilesProber
     {
         PathCounters result;
 
-        result = probe(directory, "metrics.files");
+        result = probe(directory, "metrics.service");
 
         //
         return result;