Fixed generic metrics display.

.classpath

@@ -28,5 +28,6 @@
 	<classpathentry kind="lib" path="lib/Logs/log4j-core-2.17.1.jar" sourcepath="lib/Logs/log4j-core-2.17.1-sources.jar"/>
 	<classpathentry kind="lib" path="lib/Logs/log4j-slf4j-impl-2.17.1.jar" sourcepath="lib/Logs/log4j-slf4j-impl-2.17.1-sources.jar"/>
 	<classpathentry kind="lib" path="lib/commons-io-2.11.0.jar" sourcepath="lib/commons-io-2.11.0-sources.jar"/>
+	<classpathentry kind="lib" path="lib/json-simple-1.1.1.jar" sourcepath="lib/json-simple-1.1.1-sources.jar"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>

CONCEPT.md

@@ -65,8 +65,8 @@ Voici une liste de types simples :
 
 | Type | Description | Modèle | Exemple |
 | ------ | ------ | ------ | ------ |
-| DATE | Une date au format ISO | ISO_DATE  | 2020-07-06 |
-| DATETIME | Une date horaire au format ISO | ISO_DATETIME  | 2020-07-06T14:23:20 |
+| DATE | Une date au format ISO | ISO_8601 | 2020-07-06 |
+| DATETIME | Une date horaire au format ISO | ISO_8601 | 2020-07-06T14:23:20 |
 | DAYS | Une liste de 366 valeurs numériques avec séparateurs | STU | 1;2;3;4;5;…;365 |
 | EMAIL | Un courriel. | ^.+@.+$ | christian@momon.org |
 | MONTHS | Une liste de 12 valeurs numériques avec séparateurs | STU | 1;2;3;4;5;6;7;9;10;11;12 |

ONTOLOGY.md

@@ -9,7 +9,7 @@ Cette section regroupe des informations sur le fichier *properties*.
 file.class = federation
 
 # Version de l'ontologie utilisée utilisé (type STRING, recommandé).
-file.protocol = ChatonsInfos-0.1
+file.protocol = ChatonsInfos-0.5
 
 # Date et horaire de génération du fichier (type DATETIME, recommandé).
 file.datetime = 2020-07-06T14:23:20

README.md

@@ -164,6 +164,7 @@ Configuration template:
 ```
 conf.probe.types=PrivateBin
 conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
+conf.probe.httpaccesslog.pattern=
 conf.probe.privatebin.data=/var/www/paste.libre-service.eu/data/
 conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
 ```
@@ -176,12 +177,18 @@ Configuration template:
 conf.probe.types=Framadate
 conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
 ```
-### Gitea metrics (coming soon)
+
+### Gitea metrics
 
 Configuration template:
 
 ```
 conf.probe.types=Gitea
+conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
+conf.probe.httpaccesslog.pattern=
+conf.probe.gitea.data=/opt/gitea/data
+conf.probe.gitea.api.url=https://forge.libre-service.eu/
+conf.probe.gitea.token=b6598c616b1cd350b834258205da4e5e8b951005
 conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
 ```
 
@@ -192,6 +199,7 @@ Configuration template:
 ```
 conf.probe.types=LibreQR
 conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
+conf.probe.httpaccesslog.pattern=
 conf.probe.libreqr.data=/var/www/qrcode.libre-service.eu/data/
 conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
 ```
@@ -203,6 +211,7 @@ Configuration template:
 ```
 conf.probe.types=Nextcloud
 conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
+conf.probe.httpaccesslog.pattern=
 conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
 ```
 

resources/conf/sample-web.conf

@@ -4,7 +4,7 @@
 # Warning: move this file outside the Git directory.
 
 conf.class=federation
-conf.protocol=StatoolInfos-0.4.0
+conf.protocol=StatoolInfos-0.5.0
 
 conf.crawl.input=https://www.libre-service.eu/.well-known/statoolinfos/libre-service.eu.properties
 conf.crawl.cache=/srv/statoolinfos/cache/

snapshot

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+cd $(dirname "$0")
+
 ./build.sh -snapshot

src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -197,6 +197,30 @@ public final class StatoolInfosCLI
         return result;
     }
 
+    /**
+     * Normalize bash parameter.
+     *
+     * @param value
+     *            the value
+     * @return the string
+     */
+    private static String normalizeBashParameter(final String value)
+    {
+        String result;
+
+        if (value == null)
+        {
+            result = null;
+        }
+        else
+        {
+            result = StringUtils.trim(value).replace("\\*", "*").replace("\\?", "?");
+        }
+
+        //
+        return result;
+    }
+
     /**
      * Parses the log filter option.
      *
@@ -508,6 +532,23 @@ public final class StatoolInfosCLI
             }
             System.out.println(chrono.format());
         }
+        else if (isMatching(args, "testHttpAccessLog", "\\s*\\S+\\s*"))
+        {
+            System.out.println("param1=" + args[1]);
+            String source = normalizeBashParameter(args[1]);
+            System.out.println("source=" + source);
+            Files files = FilesUtils.searchByWildcard(source);
+            System.out.println(files.size());
+            try
+            {
+                StatoolInfos.testHttpAccessLog(files);
+            }
+            catch (Exception exception)
+            {
+                logger.error("Error: {}", exception.getMessage());
+                exception.printStackTrace();
+            }
+        }
         else
         {
             System.out.println("Bad usage.");

src/fr/devinsy/statoolinfos/core/Federation.java

@@ -135,6 +135,21 @@ public class Federation extends PathPropertyList
         return result;
     }
 
+    /**
+     * Gets the away organizations.
+     *
+     * @return the away organizations
+     */
+    public Organizations getAwayOrganizations()
+    {
+        Organizations result;
+
+        result = this.organizations.filterAwayFor(getTechnicalName());
+
+        //
+        return result;
+    }
+
     /**
      * Gets the contact email.
      *
@@ -220,6 +235,21 @@ public class Federation extends PathPropertyList
         return result;
     }
 
+    /**
+     * Gets the idle organizations.
+     *
+     * @return the idle organizations
+     */
+    public Organizations getIdleOrganizations()
+    {
+        Organizations result;
+
+        result = this.organizations.getIdles();
+
+        //
+        return result;
+    }
+
     public PropertyChecks getInputChecks()
     {
         return this.inputChecks;
@@ -339,6 +369,21 @@ public class Federation extends PathPropertyList
         return result;
     }
 
+    /**
+     * Gets the member organizations.
+     *
+     * @return the member organizations
+     */
+    public Organizations getMemberOrganizations()
+    {
+        Organizations result;
+
+        result = this.organizations.filterMemberFor(getTechnicalName());
+
+        //
+        return result;
+    }
+
     /**
      * Gets the metric month values all.
      *

src/fr/devinsy/statoolinfos/core/Organization.java

@@ -86,6 +86,28 @@ public class Organization extends PathPropertyList
         this.crawlJournal = new CrawlJournal();
     }
 
+    /**
+     * Gets the service active count.
+     *
+     * @return the service active count
+     */
+    public int getActiveServiceCount()
+    {
+        int result;
+
+        result = 0;
+        for (Service service : this.services)
+        {
+            if (service.isActive())
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
     /**
      * Gets the active services.
      *
@@ -109,28 +131,6 @@ public class Organization extends PathPropertyList
         return result;
     }
 
-    /**
-     * Gets the service active count.
-     *
-     * @return the service active count
-     */
-    public int getActiveServiceCount()
-    {
-        int result;
-
-        result = 0;
-        for (Service service : this.services)
-        {
-            if (service.isActive())
-            {
-                result += 1;
-            }
-        }
-
-        //
-        return result;
-    }
-
     /**
      * Gets the age.
      *
@@ -1074,6 +1074,29 @@ public class Organization extends PathPropertyList
         return result;
     }
 
+    /**
+     * Checks if is idle.
+     *
+     * @return true, if is idle
+     */
+    public boolean isIdle()
+    {
+        boolean result;
+
+        Status status = getStatus();
+        if ((status == null) || (status == Status.IDLE))
+        {
+            result = true;
+        }
+        else
+        {
+            result = false;
+        }
+
+        //
+        return result;
+    }
+
     /**
      * Checks if is valid.
      *

src/fr/devinsy/statoolinfos/core/Organizations.java

@@ -36,6 +36,29 @@ public class Organizations extends ArrayList<Organization>
         super();
     }
 
+    /**
+     * Filter active.
+     *
+     * @return the organizations
+     */
+    public Organizations filterActive()
+    {
+        Organizations result;
+
+        result = new Organizations();
+
+        for (Organization organization : this)
+        {
+            if (organization.isActive())
+            {
+                result.add(organization);
+            }
+        }
+
+        //
+        return result;
+    }
+
     /**
      * Filter member of.
      *
@@ -51,7 +74,32 @@ public class Organizations extends ArrayList<Organization>
 
         for (Organization organization : this)
         {
-            if ((!organization.isAway()) && (!organization.isAwayFor(entityName)))
+            if ((organization.isActive()) && (!organization.isAwayFor(entityName)))
+            {
+                result.add(organization);
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Filter away for.
+     *
+     * @param entityName
+     *            the entity name
+     * @return the organizations
+     */
+    public Organizations filterAwayFor(final String entityName)
+    {
+        Organizations result;
+
+        result = new Organizations();
+
+        for (Organization organization : this)
+        {
+            if ((organization.isAway()) || (organization.isAwayFor(entityName)))
             {
                 result.add(organization);
             }
@@ -109,6 +157,31 @@ public class Organizations extends ArrayList<Organization>
         return result;
     }
 
+    /**
+     * Filter member for.
+     *
+     * @param entityName
+     *            the entity name
+     * @return the organizations
+     */
+    public Organizations filterMemberFor(final String entityName)
+    {
+        Organizations result;
+
+        result = new Organizations();
+
+        for (Organization organization : this)
+        {
+            if ((!organization.isAway()) && (!organization.isAwayFor(entityName)))
+            {
+                result.add(organization);
+            }
+        }
+
+        //
+        return result;
+    }
+
     /**
      * Gets the active service count.
      *
@@ -128,6 +201,29 @@ public class Organizations extends ArrayList<Organization>
         return result;
     }
 
+    /**
+     * Gets the idles.
+     *
+     * @return the idles
+     */
+    public Organizations getIdles()
+    {
+        Organizations result;
+
+        result = new Organizations();
+
+        for (Organization organization : this)
+        {
+            if (organization.isIdle())
+            {
+                result.add(organization);
+            }
+        }
+
+        //
+        return result;
+    }
+
     /**
      * Gets the service count.
      *

src/fr/devinsy/statoolinfos/core/StatoolInfos.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -34,7 +34,8 @@ import fr.devinsy.statoolinfos.crawl.Crawler;
 import fr.devinsy.statoolinfos.htmlize.Htmlizer;
 import fr.devinsy.statoolinfos.metrics.Prober;
 import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
-import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogIterator;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogParser;
 import fr.devinsy.statoolinfos.properties.PathProperties;
 import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
 import fr.devinsy.statoolinfos.stats.ip.IpStat;
@@ -46,6 +47,7 @@ import fr.devinsy.statoolinfos.stats.visitor.VisitorStator;
 import fr.devinsy.statoolinfos.uptime.UptimeJournal;
 import fr.devinsy.statoolinfos.uptime.UptimeSurveyor;
 import fr.devinsy.statoolinfos.util.Chrono;
+import fr.devinsy.statoolinfos.util.Files;
 import fr.devinsy.statoolinfos.util.LineIterator;
 import fr.devinsy.strings.StringList;
 import fr.devinsy.strings.StringsUtils;
@@ -180,7 +182,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
+                HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
 
                 if (filter.matches(log))
                 {
@@ -230,7 +232,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
+                HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
 
                 if (filter.matches(log))
                 {
@@ -281,7 +283,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
+                HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
 
                 if (filter.matches(log))
                 {
@@ -347,7 +349,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
+                HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
                 if (filter.matches(log))
                 {
                     stator.putLog(log);
@@ -398,7 +400,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
+                HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
                 if (filter.matches(log))
                 {
                     stator.putLog(log);
@@ -449,7 +451,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
+                HttpAccessLog log = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
                 if (filter.matches(log))
                 {
                     stator.putLog(log);
@@ -531,6 +533,22 @@ public class StatoolInfos
         }
     }
 
+    /**
+     * Test http access log.
+     *
+     * @param source
+     *            the source
+     * @throws IOException
+     */
+    public static void testHttpAccessLog(final Files source) throws IOException
+    {
+        HttpAccessLogIterator logs = new HttpAccessLogIterator(source);
+        while (logs.hasNext())
+        {
+            System.out.println(logs.next().toStringLog());
+        }
+    }
+
     /**
      * Uptime.
      *

src/fr/devinsy/statoolinfos/crawl/Crawler.java

@@ -105,9 +105,9 @@ public class Crawler
     {
         logger.info("Crawling {}", url);
 
+        File downloadFile = null;
         try
         {
-            File downloadFile;
             try
             {
                 downloadFile = download(url);
@@ -158,6 +158,7 @@ public class Crawler
                     {
                         logger.error("ERROR: bad child class [{}][{}].", downloadClass, parent);
                         this.journal.add(url, parentURL, CrawlStatus.BADCHILDCLASS);
+                        downloadFile.delete();
                     }
                     else
                     {
@@ -177,6 +178,7 @@ public class Crawler
                         if (StringUtils.equals(downloadSha, storedSha))
                         {
                             this.journal.add(url, parentURL, CrawlStatus.SUCCESS);
+                            downloadFile.delete();
                         }
                         else
                         {
@@ -234,6 +236,13 @@ public class Crawler
         {
             this.journal.add(url, parentURL, CrawlStatus.IOERROR);
         }
+        finally
+        {
+            if ((downloadFile != null) && (downloadFile.exists()))
+            {
+                downloadFile.delete();
+            }
+        }
     }
 
     /**
@@ -325,6 +334,7 @@ public class Crawler
         {
             final int TIMEOUT = 5000;
             result = Files.createTempFile("tmp-", ".statoolsinfos").toFile();
+            result.deleteOnExit();
             FileUtils.copyURLToFile(url, result, TIMEOUT, TIMEOUT);
         }
 

src/fr/devinsy/statoolinfos/htmlize/FederationMetricGenericPage.java

@@ -71,13 +71,16 @@ public class FederationMetricGenericPage
             //
             FederationMetricHtmlizer.htmlize(data, "graphic.users", federation, view, period, "metrics.service.users", ChartColor.GREEN);
 
-            FederationMetricHtmlizer.htmlize(data, "graphic.accounts.combo", federation, view, period, "Accounts + active", "metrics.service.accounts", ChartColor.YELLOW,
-                    "metrics.service.accounts.active", ChartColor.GREEN);
+            FederationMetricHtmlizer.htmlize(data, "graphic.users.ipv4-ipv6", federation, view, period, "graphic.users (ipv4 + ipv6)", "metrics.service.users.ipv4", ChartColor.YELLOW,
+                    "metrics.service.users.ipv6", ChartColor.GREEN);
+            FederationMetricHtmlizer.htmlize(data, "graphic.users.ipv4", federation, view, period, "metrics.service.users.ipv4", ChartColor.YELLOW);
+            FederationMetricHtmlizer.htmlize(data, "graphic.users.ipv6", federation, view, period, "metrics.service.users.ipv6", ChartColor.GREEN);
+
             FederationMetricHtmlizer.htmlize(data, "graphic.accounts", federation, view, period, "metrics.service.accounts", ChartColor.GREEN);
             FederationMetricHtmlizer.htmlize(data, "graphic.accounts.active", federation, view, period, "metrics.service.accounts.active", ChartColor.GREEN);
 
             FederationMetricHtmlizer.htmlize(data, "graphic.database.bytes", federation, view, period, "metrics.service.database.bytes", ChartColor.GREEN);
-            FederationMetricHtmlizer.htmlize(data, "graphic.files.bytes", federation, view, period, "metrics.service.files.bytes", ChartColor.GREEN);
+            FederationMetricHtmlizer.htmlize(data, "graphic.datafiles.bytes", federation, view, period, "metrics.service.datafiles.bytes", ChartColor.GREEN);
 
             //
             String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/serviceMetricGenericView.xhtml", data).toString();

src/fr/devinsy/statoolinfos/htmlize/FederationOrganizationsPage.java

@@ -78,7 +78,8 @@ public class FederationOrganizationsPage
             TagDataManager data = new TagDataManager();
 
             data.setContent("federationHeaderView", FederationHeaderView.htmlize(federation));
-            data.setContent("organizationListView", OrganizationListView.htmlize(federation.getActiveOrganizations()));
+
+            data.setContent("organizationListView", OrganizationListView.htmlize(federation.getMemberOrganizations(), federation.getAwayOrganizations()));
 
             String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/federationOrganizations.xhtml", data).toString();
 

src/fr/devinsy/statoolinfos/htmlize/OrganizationListView.java

@@ -29,6 +29,7 @@ import fr.devinsy.statoolinfos.core.Organization;
 import fr.devinsy.statoolinfos.core.Organizations;
 import fr.devinsy.statoolinfos.core.StatoolInfosException;
 import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
+import fr.devinsy.strings.StringList;
 import fr.devinsy.xidyn.XidynException;
 import fr.devinsy.xidyn.data.TagDataManager;
 import fr.devinsy.xidyn.presenters.PresenterUtils;
@@ -52,7 +53,7 @@ public class OrganizationListView
      * @throws IOException
      *             Signals that an I/O exception has occurred.
      */
-    public static String htmlize(final Organizations organizations) throws StatoolInfosException, IOException
+    public static String htmlize(final Organizations organizations, final Organizations aways) throws StatoolInfosException, IOException
     {
         String result;
 
@@ -60,8 +61,9 @@ public class OrganizationListView
         {
             TagDataManager data = new TagDataManager();
 
-            data.setContent("organizationCount", organizations.size());
-            data.setContent("serviceCount", organizations.getActiveServiceCount());
+            Organizations actives = organizations.filterActive();
+            data.setContent("organizationCount", actives.size());
+            data.setContent("serviceCount", actives.getActiveServiceCount());
 
             String monthLabel = LocalDate.now().minusMonths(1).format(DateTimeFormatter.ofPattern("MMMM yyyy")).replace(" ", " ");
             data.setContent("monthLabel", monthLabel);
@@ -69,7 +71,7 @@ public class OrganizationListView
             data.setAttribute("visitCountHeaderColumn", "title", monthLabel);
 
             int index = 0;
-            for (Organization organization : organizations.sortByReverseServiceCount())
+            for (Organization organization : actives.sortByReverseServiceCount())
             {
                 data.setAttribute("organizationListLine", index, "organizationListLineNameLink", "href", organization.getTechnicalName() + ".xhtml");
                 data.setAttribute("organizationListLine", index, "organizationListLineLogo", "src", organization.getLogoFileName());
@@ -91,6 +93,35 @@ public class OrganizationListView
                 index += 1;
             }
 
+            Organizations idles = organizations.getIdles();
+            if (idles.isEmpty())
+            {
+                data.setContent("idleMembers", "aucun.");
+            }
+            else
+            {
+                StringList buffer = new StringList();
+                for (Organization idle : idles)
+                {
+                    buffer.add(String.format("<a href=\"%s\" style=\"text-decoration: none;\">%s</a>", idle.getTechnicalName() + ".xhtml", idle.getName()));
+                }
+                data.setContent("idleMembers", buffer.toString("", ", ", "."));
+            }
+
+            if (aways.isEmpty())
+            {
+                data.setContent("awayMembers", "aucun.");
+            }
+            else
+            {
+                StringList buffer = new StringList();
+                for (Organization away : aways)
+                {
+                    buffer.add(String.format("<a href=\"%s\" style=\"text-decoration: none;\">%s</a>", away.getTechnicalName() + ".xhtml", away.getName()));
+                }
+                data.setContent("awayMembers", buffer.toString("", ", ", "."));
+            }
+
             String page = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/organizationListView.xhtml", data).toString();
 
             result = XidynUtils.extractBodyContent(page);

src/fr/devinsy/statoolinfos/htmlize/OrganizationMetricGenericPage.java

@@ -72,12 +72,15 @@ public class OrganizationMetricGenericPage
             //
             OrganizationMetricHtmlizer.htmlize(data, "graphic.users", organization, view, period, "metrics.service.users", ChartColor.GREEN);
 
-            OrganizationMetricHtmlizer.htmlize(data, "graphic.accounts.combo", organization, view, period, "Accounts + active", "metrics.service.accounts", ChartColor.YELLOW,
-                    "metrics.service.accounts.active", ChartColor.GREEN);
+            OrganizationMetricHtmlizer.htmlize(data, "graphic.users.ipv4-ipv6", organization, view, period, "graphic.users (ipv4 + ipv6)", "metrics.service.users.ipv4", ChartColor.YELLOW,
+                    "metrics.service.users.ipv6", ChartColor.GREEN);
+            OrganizationMetricHtmlizer.htmlize(data, "graphic.users.ipv4", organization, view, period, "metrics.service.users.ipv4", ChartColor.YELLOW);
+            OrganizationMetricHtmlizer.htmlize(data, "graphic.users.ipv6", organization, view, period, "metrics.service.users.ipv6", ChartColor.GREEN);
+
             OrganizationMetricHtmlizer.htmlize(data, "graphic.accounts", organization, view, period, "metrics.service.accounts", ChartColor.GREEN);
             OrganizationMetricHtmlizer.htmlize(data, "graphic.accounts.active", organization, view, period, "metrics.service.accounts.active", ChartColor.GREEN);
             OrganizationMetricHtmlizer.htmlize(data, "graphic.database.bytes", organization, view, period, "metrics.service.database.bytes", ChartColor.GREEN);
-            OrganizationMetricHtmlizer.htmlize(data, "graphic.files.bytes", organization, view, period, "metrics.service.files.bytes", ChartColor.GREEN);
+            OrganizationMetricHtmlizer.htmlize(data, "graphic.datafiles.bytes", organization, view, period, "metrics.service.datafiles.bytes", ChartColor.GREEN);
 
             //
             String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/serviceMetricGenericView.xhtml", data).toString();

src/fr/devinsy/statoolinfos/htmlize/ServiceMetricGenericPage.java

@@ -72,13 +72,16 @@ public class ServiceMetricGenericPage
             //
             ServiceMetricHtmlizer.htmlize(data, "graphic.users", service, view, period, "metrics.service.users", ChartColor.GREEN);
 
-            ServiceMetricHtmlizer.htmlize(data, "graphic.accounts.combo", service, view, period, "Accounts + active", "metrics.service.accounts", ChartColor.YELLOW,
-                    "metrics.service.accounts.active", ChartColor.GREEN);
+            ServiceMetricHtmlizer.htmlize(data, "graphic.users.ipv4-ipv6", service, view, period, "graphic.users (ipv4 + ipv6)", "metrics.service.users.ipv4", ChartColor.YELLOW,
+                    "metrics.service.users.ipv6", ChartColor.GREEN);
+            ServiceMetricHtmlizer.htmlize(data, "graphic.users.ipv4", service, view, period, "metrics.service.users.ipv4", ChartColor.YELLOW);
+            ServiceMetricHtmlizer.htmlize(data, "graphic.users.ipv6", service, view, period, "metrics.service.users.ipv6", ChartColor.GREEN);
+
             ServiceMetricHtmlizer.htmlize(data, "graphic.accounts", service, view, period, "metrics.service.accounts", ChartColor.GREEN);
             ServiceMetricHtmlizer.htmlize(data, "graphic.accounts.active", service, view, period, "metrics.service.accounts.active", ChartColor.GREEN);
 
             ServiceMetricHtmlizer.htmlize(data, "graphic.database.bytes", service, view, period, "metrics.service.database.bytes", ChartColor.GREEN);
-            ServiceMetricHtmlizer.htmlize(data, "graphic.files.bytes", service, view, period, "metrics.service.files.bytes", ChartColor.GREEN);
+            ServiceMetricHtmlizer.htmlize(data, "graphic.datafiles.bytes", service, view, period, "metrics.service.datafiles.bytes", ChartColor.GREEN);
 
             // metrics.moderation.accounts.reported.* =
             // metrics.moderation.accounts.sanctioned.* =

src/fr/devinsy/statoolinfos/htmlize/organizationListView.xhtml

@@ -13,8 +13,8 @@
 <body>
 	<div class="center_table" style="width: 900px;">
 		<br/>
-		<div class="center">Nombre de membres : <span id="organizationCount">n/a</span></div>
-		<div class="center">Nombre de services : <span id="serviceCount">n/a</span></div>
+		<div class="center">Nombre de membres actifs : <span id="organizationCount">n/a</span></div>
+		<div class="center">Nombre de services acfifs : <span id="serviceCount">n/a</span></div>
 		<table id="organizations" class="table_classic left">
 			<thead>
 				<tr>
@@ -43,6 +43,8 @@
 			</tbody>
 		</table>
 		<div id="month_asterisk">(*) chiffres de <span id="monthLabel">n/a</span></div>
+		<div id="idleMembersZone" class="left">Membres en sommeil : <span id="idleMembers">n/a</span></div>
+		<div id="awayMembersZone" class="left">Anciens membres : <span id="awayMembers">n/a</span></div>
 	</div>
 	<script type="text/javascript">
 		$(document).ready(function()

src/fr/devinsy/statoolinfos/htmlize/serviceHeaderView.xhtml

@@ -28,9 +28,9 @@
 		Inscription : 	
 			<img id="registrationNoneImg" src="circle-icons/global-mono.svg" class="disabled" title="Sans compte"/>
 			<img id="registrationFreeImg" src="circle-icons/profile-mono.svg" class="disabled" title="Compte libre"/>
-			<img id="registrationMemberImg" src="circle-icons/rgb-mono.svg" class="disabled" title="Réservé aux membres"/>
-			<img id="registrationClientImg" src="circle-icons/creditcard-mono.svg" class="disabled" title="Réservé aux clients"/>
-			<img id="registrationLoadImg" src="circle-icons/lock-mono.svg" class="disabled" title="Actuellement fermé"/>
+			<img id="registrationMemberImg" src="circle-icons/rgb-mono.svg" class="disabled" title="Réservée aux membres"/>
+			<img id="registrationClientImg" src="circle-icons/creditcard-mono.svg" class="disabled" title="Réservée aux clients"/>
+			<img id="registrationLoadImg" src="circle-icons/lock-mono.svg" class="disabled" title="Actuellement fermée"/>
 		</div>
 		<div class="content_infos" style="margin: 5px;">
 			<a id="legalLink" href="#"><img id="legalLinkImg" src="circle-icons/ribbon.svg" class="disabled" title="Mentions légales"/>Légal</a>

src/fr/devinsy/statoolinfos/htmlize/serviceListView.xhtml

@@ -93,13 +93,13 @@
 						<img id="serviceListLineRegistrationFreeImg" src="circle-icons/profile-mono.svg" class="disabled" title="Compte libre"/>
 					</td>
 					<td id="serviceListLineRegistrationMember" class="td_icon_alone">
-						<img id="serviceListLineRegistrationMemberImg" src="circle-icons/rgb-mono.svg" class="disabled" title="Réservé aux membres"/>
+						<img id="serviceListLineRegistrationMemberImg" src="circle-icons/rgb-mono.svg" class="disabled" title="Réservée aux membres"/>
 					</td>
 					<td id="serviceListLineRegistrationClient" class="td_icon_alone">
-						<img id="serviceListLineRegistrationClientImg" src="circle-icons/creditcard-mono.svg" class="disabled" title="Réservé aux clients"/>
+						<img id="serviceListLineRegistrationClientImg" src="circle-icons/creditcard-mono.svg" class="disabled" title="Réservée aux clients"/>
 					</td>
 					<td id="serviceListLineRegistrationLoad" class="td_icon_alone">
-						<img id="serviceListLineRegistrationLoadImg" src="circle-icons/lock-mono.svg" class="disabled" title="Actuellement fermé"/>
+						<img id="serviceListLineRegistrationLoadImg" src="circle-icons/lock-mono.svg" class="disabled" title="Actuellement fermée"/>
 					</td>
 
 					<td id="" class="td_icon_alone">

src/fr/devinsy/statoolinfos/htmlize/serviceMetricGenericView.xhtml

@@ -18,13 +18,16 @@
 			<h2>Utilisateurs</h2>
 			<div id="graphic.users" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
 			<br/>			
-			<div id="graphic.accounts.combo" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
+			<div id="graphic.users.ipv4-ipv6" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
+			<div id="graphic.users.ipv4" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
+			<div id="graphic.users.ipv6" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
+			<br/>			
 			<div id="graphic.accounts" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
 			<div id="graphic.accounts.active" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
 
 			<h2>Données</h2>
 			<div id="graphic.database.bytes" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
-			<div id="graphic.files.bytes" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
+			<div id="graphic.datafiles.bytes" style="width: 450px; height: 300px; display: inline-block; border: 1px solid #e7e7e7;"/>
 		</div>
 	</div>
 </body>

src/fr/devinsy/statoolinfos/metrics/PathCounters.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -210,6 +210,27 @@ public class PathCounters extends HashMap<String, PathCounter>
         return result;
     }
 
+    public StringList getNowTimeMarks()
+    {
+        StringList result;
+
+        result = new StringList();
+
+        LocalDate now = LocalDate.now();
+        String year = TimeMark.yearOf(now).toString();
+        String yearMonth = TimeMark.yearMonthOf(now).toString();
+        String yearWeek = TimeMark.yearWeekOf(now).toString();
+        String date = TimeMark.dayOf(now).toString();
+
+        result.add(year);
+        result.add(yearMonth);
+        result.add(yearWeek);
+        result.add(date);
+
+        //
+        return result;
+    }
+
     /**
      * Gets the prefixes.
      *
@@ -393,6 +414,38 @@ public class PathCounters extends HashMap<String, PathCounter>
         }
     }
 
+    /**
+     * Rename path.
+     *
+     * @param oldPath
+     *            the old path
+     * @param newPath
+     *            the new path
+     * @return the long
+     */
+    public long renamePath(final String oldPath, final String newPath)
+    {
+        long result;
+
+        result = 0;
+        for (String key : new StringList(this.keySet()))
+        {
+            if (key.startsWith(oldPath))
+            {
+                result += 1;
+
+                PathCounter counter = get(key);
+                counter.setPath(counter.getPath().replace(oldPath, newPath));
+                remove(key);
+                put(counter);
+                System.out.println("Renamed " + key + " to " + counter.getPath());
+            }
+        }
+
+        //
+        return result;
+    }
+
     /**
      * Search by date.
      *
@@ -483,6 +536,19 @@ public class PathCounters extends HashMap<String, PathCounter>
         return result;
     }
 
+    /**
+     * Sets the.
+     *
+     * @param value
+     *            the value
+     * @param path
+     *            the path
+     */
+    public void set(final long value, final String path)
+    {
+        set(value, path, getNowTimeMarks());
+    }
+
     /**
      * Sets the.
      *
@@ -522,4 +588,22 @@ public class PathCounters extends HashMap<String, PathCounter>
 
         counter.setCounter(value);
     }
+
+    /**
+     * Sets the.
+     *
+     * @param value
+     *            the value
+     * @param path
+     *            the path
+     * @param timeMarks
+     *            the time marks
+     */
+    public void set(final long value, final String path, final StringList timeMarks)
+    {
+        for (String timeMark : timeMarks)
+        {
+            set(value, path, timeMark);
+        }
+    }
 }

src/fr/devinsy/statoolinfos/metrics/Prober.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -38,8 +38,10 @@ import fr.devinsy.statoolinfos.core.DatabaseConfig;
 import fr.devinsy.statoolinfos.core.Factory;
 import fr.devinsy.statoolinfos.core.StatoolInfosException;
 import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
+import fr.devinsy.statoolinfos.metrics.gitea.GiteaProber;
 import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
 import fr.devinsy.statoolinfos.metrics.http.HttpErrorLogAnalyzer;
+import fr.devinsy.statoolinfos.metrics.libreqr.LibreQRProber;
 import fr.devinsy.statoolinfos.metrics.minetest.MinetestProber;
 import fr.devinsy.statoolinfos.metrics.mumble.MumbleProber;
 import fr.devinsy.statoolinfos.metrics.privatebin.PrivatebinProber;
@@ -47,6 +49,7 @@ import fr.devinsy.statoolinfos.properties.PathProperties;
 import fr.devinsy.statoolinfos.properties.PathProperty;
 import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
 import fr.devinsy.statoolinfos.util.BuildInformation;
+import fr.devinsy.statoolinfos.util.FilesUtils;
 import fr.devinsy.strings.StringList;
 import fr.devinsy.strings.StringsUtils;
 
@@ -109,7 +112,7 @@ public class Prober
             //
             if (types.containsAnyIgnoreCase("HttpAccessLog"))
             {
-                logger.info("== Processing HttpAccessLog.");
+                logger.info("== Probing HttpAccessLog.");
                 String source = configuration.getProbeHttpAccessLogSource();
                 String patternRegex = configuration.getProbeHttpAccessLogPattern();
                 logger.info("source=[{}]", source);
@@ -122,7 +125,7 @@ public class Prober
             //
             if (types.containsAnyIgnoreCase("HttpErrorLog"))
             {
-                logger.info("== Processing HttpErrorLog.");
+                logger.info("== Probing HttpErrorLog.");
                 String source = configuration.getProbeHttpErrorLogSource();
                 logger.info("source=[{}]", source);
 
@@ -133,18 +136,56 @@ public class Prober
             //
             if (types.containsAnyIgnoreCase("Framadate"))
             {
-                logger.info("== Processing Framadate.");
-                String source = configuration.getProbeHttpErrorLogSource();
+                logger.info("== Probing Framadate.");
+                String source = configuration.getProbeHttpAccessLogSource();
                 logger.info("source=[{}]", source);
 
-                // PathCounters data = HttpErrorLogAnalyzer.probe(source);
+                // PathCounters datafilesPath =
+                // HttpErrorLogAnalyzer.probe(source);
                 // counters.putAll(data);
             }
 
+            //
+            if (types.containsAnyIgnoreCase("Gitea"))
+            {
+                logger.info("== Probing Gitea.");
+                String httpLogs = configuration.getProbeHttpAccessLogSource();
+                logger.info("httpLogs=[{}]", httpLogs);
+                String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
+                logger.info("httpAccessPattern=[{}]", httpAccessLogRegex);
+                File dataDirectory = configuration.getAsFile("conf.probe.gitea.data");
+                logger.info("dataPath=[{}]", dataDirectory);
+                String apiURL = configuration.get("conf.probe.gitea.api.url");
+                logger.info("apiURL=[{}]", apiURL);
+                String apiToken = configuration.get("conf.probe.gitea.api.token");
+                logger.info("apiToken=[{}]", apiToken == null ? "null" : "************************");
+
+                PathCounters metrics = GiteaProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, apiURL, apiToken, dataDirectory);
+                counters.putAll(metrics);
+            }
+
+            //
+            if (types.containsAnyIgnoreCase("LibreQR"))
+            {
+                logger.info("== Probing LibreQR.");
+
+                String source = configuration.getProbeHttpAccessLogSource();
+                logger.info("source=[{}]", source);
+
+                String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
+                logger.info("pattern=[{}]", httpAccessLogRegex);
+
+                File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles");
+                logger.info("dataDirectory=[{}]", dataDirectory);
+
+                PathCounters data = LibreQRProber.probe(FilesUtils.searchByWildcard(source), httpAccessLogRegex, dataDirectory);
+                counters.putAll(data);
+            }
+
             //
             if (types.containsAnyIgnoreCase("Minetest"))
             {
-                logger.info("== Processing Minetest.");
+                logger.info("== Probing Minetest.");
                 String source = configuration.get("conf.probe.minetest.logs");
                 DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.minetest.players");
 
@@ -155,7 +196,7 @@ public class Prober
             //
             if (types.containsAnyIgnoreCase("Mumble"))
             {
-                logger.info("== Processing Mumble.");
+                logger.info("== Probing Mumble.");
                 String source = configuration.get("conf.probe.mumble.logs");
                 logger.info("source=[{}]", source);
 
@@ -166,15 +207,15 @@ public class Prober
             //
             if (types.containsAnyIgnoreCase("PrivateBin"))
             {
-                logger.info("== Processing Privatebin.");
+                logger.info("== Probing Privatebin.");
                 String httpAccessLogs = configuration.getProbeHttpAccessLogSource();
-                String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
-                File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
                 logger.info("source=[{}]", httpAccessLogs);
+                String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
                 logger.info("pattern=[{}]", httpAccessLogRegex);
+                File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
                 logger.info("dataDirectory=[{}]", dataDirectory);
 
-                PathCounters data = PrivatebinProber.probe(httpAccessLogs, httpAccessLogRegex, dataDirectory);
+                PathCounters data = PrivatebinProber.probe(FilesUtils.searchByWildcard(httpAccessLogs), httpAccessLogRegex, dataDirectory);
                 counters.putAll(data);
             }
 
@@ -202,6 +243,9 @@ public class Prober
                     PathCounters previousCounters = readMetrics(target);
                     logger.info("previous size={}", previousCounters.size());
 
+                    // Temporary fixed 2022-02.
+                    previousCounters.renamePath("metrics.service.files", "metrics.service.datafiles");
+
                     // Merge.
                     logger.info("== Merging");
                     for (PathCounter counter : counters.values())

src/fr/devinsy/statoolinfos/metrics/TimeMarkUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -21,6 +21,7 @@ package fr.devinsy.statoolinfos.metrics;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.YearMonth;
+import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Locale;
 import java.util.regex.Pattern;
@@ -218,6 +219,23 @@ public class TimeMarkUtils
         return result;
     }
 
+    /**
+     * Year month of.
+     *
+     * @param date
+     *            the date
+     * @return the string
+     */
+    public static String yearMonthOf(final ZonedDateTime date)
+    {
+        String result;
+
+        result = date.format(DateTimeFormatter.ofPattern("yyyy-MM", Locale.FRANCE));
+
+        //
+        return result;
+    }
+
     /**
      * Year of.
      *
@@ -235,6 +253,23 @@ public class TimeMarkUtils
         return result;
     }
 
+    /**
+     * Year of.
+     *
+     * @param date
+     *            the date
+     * @return the string
+     */
+    public static String yearOf(final ZonedDateTime date)
+    {
+        String result;
+
+        result = date.format(DateTimeFormatter.ofPattern("yyyy", Locale.FRANCE));
+
+        //
+        return result;
+    }
+
     /**
      * Year week of.
      *
@@ -251,4 +286,21 @@ public class TimeMarkUtils
         //
         return result;
     }
+
+    /**
+     * Year week of.
+     *
+     * @param date
+     *            the date
+     * @return the string
+     */
+    public static String yearWeekOf(final ZonedDateTime date)
+    {
+        String result;
+
+        result = YearWeek.from(date).toString();
+
+        //
+        return result;
+    }
 }

src/fr/devinsy/statoolinfos/metrics/gitea/GiteaAPI.java

@@ -0,0 +1,368 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.gitea;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The Class GiteaAPI.
+ */
+public class GiteaAPI
+{
+    private static Logger logger = LoggerFactory.getLogger(GiteaAPI.class);
+
+    private String url;
+    private String token;
+    private JSONArray users;
+    private JSONArray organizations;
+    private JSONArray repositories;
+
+    /**
+     * Instantiates a new gitea API.
+     */
+    public GiteaAPI(final String url, final String token) throws IOException, ParseException
+    {
+        this.url = StringUtils.removeEnd(url, "/");
+        this.token = token;
+
+        //
+        String json = IOUtils.toString(new URL(this.url + "/api/v1/admin/users?limit=100000&token=" + token), Charset.defaultCharset());
+        this.users = (JSONArray) (new JSONParser().parse(json));
+
+        //
+        json = IOUtils.toString(new URL(this.url + "/api/v1/admin/orgs?limit=100000&token=" + token), Charset.defaultCharset());
+        this.organizations = (JSONArray) (new JSONParser().parse(json));
+
+        //
+        json = IOUtils.toString(new URL(this.url + "/api/v1/repos/search?limit=100000&token=" + token), Charset.defaultCharset());
+        this.repositories = (JSONArray) ((JSONObject) (new JSONParser().parse(json))).get("data");
+    }
+
+    /**
+     * Gets the active user count.
+     *
+     * @return the active user count
+     */
+    public long getActiveUserCount()
+    {
+        long result;
+
+        result = 0;
+        LocalDateTime limit = LocalDateTime.now().minusDays(31);
+        for (Object user : this.users)
+        {
+            String lastLogin = getJSONString(user, "last_login");
+
+            if (!StringUtils.isBlank(lastLogin))
+            {
+                try
+                {
+                    LocalDateTime date = LocalDateTime.parse(lastLogin, DateTimeFormatter.ISO_DATE_TIME);
+                    if (date.isAfter(limit))
+                    {
+                        result += 1;
+                    }
+                }
+                catch (DateTimeParseException exception)
+                {
+                    System.out.println("OUPS");
+                }
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the fork repository count.
+     *
+     * @return the fork repository count
+     */
+    public long getForkRepositoryCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object repository : this.repositories)
+        {
+            if (getJSONBoolean(repository, "fork"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the mirror repository count.
+     *
+     * @return the mirror repository count
+     */
+    public long getMirrorRepositoryCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object repository : this.repositories)
+        {
+            if (getJSONBoolean(repository, "mirror"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the organization count.
+     *
+     * @return the organization count
+     */
+    public long getOrganizationCount()
+    {
+        long result;
+
+        result = this.organizations.size();
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the private organization count.
+     *
+     * @return the private organization count
+     */
+    public long getPrivateOrganizationCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object organization : this.organizations)
+        {
+            String visibility = getJSONString(organization, "visibility");
+
+            if (StringUtils.equals(visibility, "private"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the private repository count.
+     *
+     * @return the private repository count
+     */
+    public long getPrivateRepositoryCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object repository : this.repositories)
+        {
+            if (getJSONBoolean(repository, "private"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the public organization count.
+     *
+     * @return the public organization count
+     */
+    public long getPublicOrganizationCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object organization : this.organizations)
+        {
+            String visibility = getJSONString(organization, "visibility");
+
+            if (StringUtils.equals(visibility, "public"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the public repository count.
+     *
+     * @return the public repository count
+     */
+    public long getPublicRepositoryCount()
+    {
+        long result;
+
+        result = 0;
+        for (Object repository : this.repositories)
+        {
+            if (!getJSONBoolean(repository, "private"))
+            {
+                result += 1;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the repository count.
+     *
+     * @return the repository count
+     */
+    public long getRepositoryCount()
+    {
+        long result;
+
+        result = this.repositories.size();
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the user count.
+     *
+     * @return the user count
+     */
+    public long getUserCount()
+    {
+        long result;
+
+        result = this.users.size();
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the boolean.
+     *
+     * @param object
+     *            the object
+     * @param key
+     *            the key
+     * @return the boolean
+     */
+    private static boolean getJSONBoolean(final Object object, final String key)
+    {
+        Boolean result;
+
+        Object value = ((JSONObject) object).get(key);
+        if (value == null)
+        {
+            result = false;
+        }
+        else
+        {
+            result = (Boolean) value;
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the long.
+     *
+     * @param object
+     *            the object
+     * @param key
+     *            the key
+     * @return the long
+     */
+    public static long getJSONLong(final Object object, final String key)
+    {
+        long result;
+
+        Object value = ((JSONObject) object).get(key);
+        if (value == null)
+        {
+            result = 0;
+        }
+        else
+        {
+            result = (Long) value;
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the as string.
+     *
+     * @param object
+     *            the object
+     * @param key
+     *            the key
+     * @return the as string
+     */
+    public static String getJSONString(final Object object, final String key)
+    {
+        String result;
+
+        Object value = ((JSONObject) object).get(key);
+        if (value == null)
+        {
+            result = "";
+        }
+        else
+        {
+            result = (String) value;
+        }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/gitea/GiteaProber.java

@@ -0,0 +1,307 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.gitea;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.StringUtils;
+import org.json.simple.parser.ParseException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.metrics.UserCounters;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
+import fr.devinsy.statoolinfos.metrics.util.DatabaseProber;
+import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.strings.StringList;
+
+/**
+ * The Class GiteaProber.
+ */
+public class GiteaProber
+{
+    private static Logger logger = LoggerFactory.getLogger(GiteaProber.class);
+
+    /**
+     * Instantiates a new nextcloud prober.
+     */
+    public GiteaProber()
+    {
+    }
+
+    /**
+     * Probe.
+     *
+     * @param httpLogs
+     *            the http logs
+     * @param httpLogRegex
+     *            the http log regex
+     * @param dataPath
+     *            the data path
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final String apiURL, final String apiToken, final File dataPath) throws IOException, StatoolInfosException
+    {
+        PathCounters result;
+
+        // metrics.service.users
+        // metrics.service.users.ipv4
+        // metrics.service.users.ipv6
+        result = probeHttpAccessLog(httpLogs, httpLogRegex);
+
+        // metrics.service.database.bytes
+        result.putAll(DatabaseProber.probe(null));
+
+        // metrics.service.files.bytes
+        // metrics.service.files.count
+        result.putAll(DatafilesProber.probe(dataPath));
+
+        //
+        System.out.println("Probing GiteaAPI [" + apiURL + "]");
+        if (!StringUtils.isBlank(apiURL) && (!StringUtils.isBlank(apiToken)))
+        {
+            try
+            {
+                GiteaAPI gitea = new GiteaAPI(apiURL, apiToken);
+
+                // metrics.service.accounts
+                // metrics.service.accounts.active
+                result.set(gitea.getUserCount(), "metrics.service.accounts");
+                result.set(gitea.getActiveUserCount(), "metrics.service.accounts.active");
+
+                // metrics.forge.groups.* =
+                // metrics.forge.groups.private.* =
+                // metrics.forge.groups.public.* =
+                result.set(gitea.getOrganizationCount(), "metrics.forge.groups");
+                result.set(gitea.getPublicOrganizationCount(), "metrics.forge.groups.public");
+                result.set(gitea.getPrivateOrganizationCount(), "metrics.forge.groups.private");
+
+                // metrics.forge.projects.* =
+                // metrics.forge.repositories.* =
+                // metrics.forge.repositories.private.* =
+                // metrics.forge.repositories.public.* =
+                result.set(gitea.getRepositoryCount(), "metrics.forge.projects");
+                result.set(gitea.getRepositoryCount(), "metrics.forge.repositories");
+                result.set(gitea.getPublicRepositoryCount(), "metrics.forge.repositories.public");
+                result.set(gitea.getPrivateRepositoryCount(), "metrics.forge.repositories.private");
+                result.set(gitea.getForkRepositoryCount(), "metrics.forge.repositories.forks");
+                result.set(gitea.getMirrorRepositoryCount(), "metrics.forge.repositories.mirrors");
+
+                // metrics.issues.count.* =
+                // metrics.issues.issuers.*=
+                // metrics.issues.created.* =
+                // metrics.issues.closed.* =
+
+            }
+            catch (IOException | ParseException exception)
+            {
+                exception.printStackTrace();
+            }
+        }
+
+        // ///////////////////////////////////////
+
+        // # [Metrics spécifiques aux services de forges logicielles].
+        //
+        //
+        // metrics.forge.commits.* =
+        // metrics.forge.mergerequests.* =
+        // metrics.forge.committers.* =
+        // metrics.forge.mergerequesters.* =
+        // metrics.forge.files.* =
+
+        //
+        return result;
+    }
+
+    /**
+     * Probe http access log.
+     *
+     * @param httpAccessLogFiles
+     *            the http access log files
+     * @param httpRegex
+     *            the http regex
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
+    {
+        PathCounters result;
+
+        result = new PathCounters();
+
+        // metrics.service.users
+        // metrics.service.users.ipv4
+        // metrics.service.users.ipv6
+        UserCounters users = new UserCounters();
+        UserCounters ipv4Users = new UserCounters();
+        UserCounters ipv6Users = new UserCounters();
+
+        StringList regexList = new StringList();
+
+        /*
+        GET /admin
+        GET /admin/auths
+        GET /admin/config
+        GET /admin/emails
+        GET /admin/hooks
+        GET /admin/monitor
+        GET /admin/notices
+        GET /admin/orgs
+        GET /admin/repos
+        GET /admin/users
+        */
+        regexList.add("GET /admin(/auths|/config|/emails|/hooks|/monitor|/notices|/orgs|/repos|/users)? .*");
+
+        /*
+        GET /cpm
+        GET /devinsy
+        */
+
+        /*
+        GET /devinsy/bacasable
+        GET /devinsy/bacasable/activity
+        GET /devinsy/bacasable/commits/branch/master
+        GET /devinsy/bacasable/compare/master...master
+        GET /devinsy/bacasable/graph
+        GET /devinsy/bacasable/issues
+        GET /devinsy/bacasable/issues/new?project=1
+        GET /devinsy/bacasable/projects
+        GET /devinsy/bacasable/projects/1
+        GET /devinsy/bacasable/projects/1/edit
+        GET /devinsy/bacasable/pulls
+        GET /devinsy/bacasable/pulls/1
+        GET /devinsy/bacasable/releases
+        GET /devinsy/bacasable/releases/tag/0.5.0
+        GET /devinsy/bacasable/tags
+        GET /devinsy/bacasable/wiki
+        GET /devinsy/bacasable/wiki/Home
+        */
+        regexList.add("GET /\\S+/\\S+(/activity|/commits/branch/master|/compare/\\S+|/graph|/issues|/projects(/\\d+)|/pulls(/\\d+)|/releases(/tag/\\S+|/tags|/wiki|/wiki/\\S+)) .*");
+
+        /*
+        GET /devinsy/bacasable/settings
+        GET /devinsy/bacasable/settings/branches
+        GET /devinsy/bacasable/settings/collaboration
+        GET /devinsy/bacasable/settings/hooks
+        GET /devinsy/bacasable/settings/keys
+        GET /devinsy/bacasable/settings/lfs
+        GET /devinsy/bacasable/settings/lfs/locks
+        GET /devinsy/bacasable/settings/lfs/pointers
+        GET /devinsy/bacasable/settings/tags
+         */
+        regexList.add("GET /\\S+/\\S+/settings(/branches|/collaboration|/hooks|/keys|/lfs|/lfs/locks|/lfs/pointers|/tags)? .*");
+
+        /*
+        GET /org/create
+        GET /org/devinsy/members
+        GET /org/devinsy/settings
+        GET /org/devinsy/teams
+        GET /org/devinsy/teams/new
+         */
+        regexList.add("GET /org/(create|/\\S+/members|/\\S+/settings|/\\S+/teams|/\\S+/teams/new) .*");
+
+        /*
+        GET /explore/organizations
+        GET /explore/repos
+        GET /explore/users
+        */
+        regexList.add("GET (/explore/organizations|/explore/repos|/explore/users) .*");
+
+        /*
+        GET /issues
+        GET /milestones
+        GET /pulls
+         */
+        regexList.add("GET /(issues|milestones|pulls) .*");
+
+        /*
+        GET /repo/create
+        GET /repo/fork/7
+        GET /repo/migrate
+        */
+        regexList.add("GET /repo/(create|fork/\\d+|migrate) .*");
+
+        /*
+        GET /user/settings
+        GET /user/settings/account
+        GET /user/settings/appearance
+        GET /user/settings/applications
+        GET /user/settings/keys
+        GET /user/settings/organization
+        GET /user/settings/repos
+        GET /user/settings/security
+         */
+        regexList.add("GET /user/settings(/account|/appearance|/applications|/keys|/organizations|/repos|/security)? .*");
+
+        String regex = regexList.toString("(", "|", ")");
+
+        Pattern USE_PATTERN = Pattern.compile(regex);
+
+        //
+        for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
+        {
+            // General HTTP access logs.
+            String year = log.getYear();
+            String yearMonth = log.getYearMonth();
+            String yearWeek = log.getYearWeek();
+            String date = log.getDate();
+
+            // metrics.service.users
+            // metrics.service.users.ipv4
+            // metrics.service.users.ipv6
+            if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
+            {
+                String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
+
+                users.put(key, year, yearMonth, yearWeek, date);
+
+                if (log.isIPv4())
+                {
+                    ipv4Users.put(key, year, yearMonth, yearWeek, date);
+                }
+                else
+                {
+                    ipv6Users.put(key, year, yearMonth, yearWeek, date);
+                }
+            }
+        }
+
+        //
+        result.putAll(users.getCounters("metrics.service.users"));
+        result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
+        result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -19,6 +19,8 @@
 package fr.devinsy.statoolinfos.metrics.http;
 
 import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Locale;
 import java.util.regex.Pattern;
@@ -40,7 +42,7 @@ public class HttpAccessLog
 
     private String ip;
     private String remoteUser;
-    private LocalDateTime time;
+    private ZonedDateTime time;
     private String request;
     private HttpStatus status;
     private long bodyBytesSent;
@@ -134,7 +136,7 @@ public class HttpAccessLog
 
     public LocalDateTime getTime()
     {
-        return this.time;
+        return this.time.toLocalDateTime();
     }
 
     public UserAgent getUserAgent()
@@ -317,7 +319,18 @@ public class HttpAccessLog
         this.status = status;
     }
 
+    /**
+     * Sets the time.
+     *
+     * @param time
+     *            the new time
+     */
     public void setTime(final LocalDateTime time)
+    {
+        this.time = ZonedDateTime.of(time, ZoneId.systemDefault());
+    }
+
+    public void setTime(final ZonedDateTime time)
     {
         this.time = time;
     }
@@ -353,4 +366,34 @@ public class HttpAccessLog
         //
         return result;
     }
+
+    /**
+     * To string log.
+     *
+     * @return the string
+     */
+    public String toStringLog()
+    {
+        String result;
+
+        // "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+)
+        // \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+)
+        // (?<bodyBytesSent>\\d+) \"(?<referer>.*)\"
+        // \"(?<userAgent>[^\"]*)\".*$");
+
+        // result = String.format("%s %s %s \\[%s\\] \"%s\" %d %d \"%s\"
+        // \"%s\"",
+        result = String.format("%s - %s [%s] \"%s\" %d %d \"%s\" \"%s\"",
+                this.ip,
+                this.remoteUser,
+                this.time.format(DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z", Locale.ENGLISH)),
+                this.request,
+                this.status.getCode(),
+                this.bodyBytesSent,
+                this.referer,
+                this.userAgent);
+
+        //
+        return result;
+    }
 }

src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java

@@ -20,9 +20,6 @@ package fr.devinsy.statoolinfos.metrics.http;
 
 import java.io.File;
 import java.io.IOException;
-import java.time.LocalDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -43,12 +40,6 @@ public class HttpAccessLogAnalyzer
 {
     private static Logger logger = LoggerFactory.getLogger(HttpAccessLogAnalyzer.class);
 
-    // log_format combined '$remote_addr - $remote_user [$time_local] '
-    // '"$request" $status $body_bytes_sent '
-    // '"$http_referer" "$http_user_agent"';
-    public static final Pattern COMBINED_PATTERN = Pattern.compile(
-            "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>.*)\" \"(?<userAgent>[^\"]*)\".*$");
-
     private int errorCount;
     private PathCounters counters;
     private IpCounters ips;
@@ -149,7 +140,7 @@ public class HttpAccessLogAnalyzer
         Pattern pattern;
         if (patternRegex == null)
         {
-            pattern = COMBINED_PATTERN;
+            pattern = HttpAccessLogParser.COMBINED_PATTERN;
         }
         else
         {
@@ -164,7 +155,7 @@ public class HttpAccessLogAnalyzer
 
             try
             {
-                HttpAccessLog log = parseLog(line, pattern);
+                HttpAccessLog log = HttpAccessLogParser.parseLog(line, pattern);
                 if (log == null)
                 {
                     logger.warn("LINE IS NOT MATCHING [{}]", line);
@@ -410,39 +401,6 @@ public class HttpAccessLogAnalyzer
         return result;
     }
 
-    /**
-     * Parses the log.
-     *
-     * @param line
-     *            the line
-     * @return the http log
-     */
-    public static HttpAccessLog parseLog(final String line, final Pattern pattern)
-    {
-        HttpAccessLog result;
-
-        Matcher matcher = pattern.matcher(line);
-        if (matcher.matches())
-        {
-            result = new HttpAccessLog();
-            result.setIp(matcher.group("remoteAddress"));
-            result.setRemoteUser(matcher.group("remoteUser"));
-            result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
-            result.setRequest(matcher.group("request"));
-            result.setStatus(HttpStatusTable.instance().get(matcher.group("status")));
-            result.setBodyBytesSent(Long.valueOf(matcher.group("bodyBytesSent")));
-            result.setReferer(matcher.group("referer"));
-            result.setUserAgent(new UserAgent(matcher.group("userAgent")));
-        }
-        else
-        {
-            result = null;
-        }
-
-        //
-        return result;
-    }
-
     /**
      * Probe.
      *

src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java

@@ -0,0 +1,159 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.http;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.statoolinfos.util.FilesLineIterator;
+
+/**
+ * The Class HttpAccessLogIterator.
+ */
+public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
+{
+    private static Logger logger = LoggerFactory.getLogger(HttpAccessLogIterator.class);
+
+    private FilesLineIterator lineIterator;
+    private Pattern pattern;
+    private HttpAccessLog nextLog;
+
+    /**
+     * Instantiates a new http access log iterator.
+     *
+     * @param source
+     *            the source
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public HttpAccessLogIterator(final Files source) throws IOException
+    {
+        this(source, null);
+    }
+
+    /**
+     * Instantiates a new http log iterator.
+     *
+     * @param source
+     *            the source
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public HttpAccessLogIterator(final Files source, final String regex) throws IOException
+    {
+        this.lineIterator = new FilesLineIterator(source);
+        this.nextLog = null;
+
+        if (regex == null)
+        {
+            this.pattern = HttpAccessLogParser.COMBINED_PATTERN;
+        }
+        else
+        {
+            this.pattern = Pattern.compile(regex);
+        }
+    }
+
+    /* (non-Javadoc)
+     * @see java.util.Iterator#hasNext()
+     */
+    @Override
+    public boolean hasNext()
+    {
+        boolean result;
+
+        preload();
+
+        if (this.nextLog == null)
+        {
+            result = false;
+        }
+        else
+        {
+            result = true;
+        }
+
+        //
+        return result;
+    }
+
+    /* (non-Javadoc)
+     * @see java.util.Iterator#next()
+     */
+    @Override
+    public HttpAccessLog next()
+    {
+        HttpAccessLog result;
+
+        preload();
+
+        result = this.nextLog;
+        this.nextLog = null;
+
+        //
+        return result;
+    }
+
+    /**
+     * Forward.
+     */
+    private void preload()
+    {
+        if (this.nextLog == null)
+        {
+            boolean ended = false;
+            while (!ended)
+            {
+                if (this.lineIterator.hasNext())
+                {
+                    String line = this.lineIterator.next();
+
+                    try
+                    {
+                        HttpAccessLog log = HttpAccessLogParser.parseLog(line, this.pattern);
+                        if (log == null)
+                        {
+                            logger.warn("LINE IS NOT MATCHING [{}]", line);
+                        }
+                        else
+                        {
+                            this.nextLog = log;
+                            ended = true;
+                        }
+                    }
+                    catch (Exception exception)
+                    {
+                        logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
+                        exception.printStackTrace();
+                    }
+                }
+                else
+                {
+                    this.nextLog = null;
+                    ended = true;
+                }
+            }
+        }
+    }
+}

src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogParser.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.http;
+
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The Class HttpAccessLogParser.
+ */
+public class HttpAccessLogParser
+{
+    private static Logger logger = LoggerFactory.getLogger(HttpAccessLogParser.class);
+
+    // log_format combined '$remote_addr - $remote_user [$time_local] '
+    // '"$request" $status $body_bytes_sent '
+    // '"$http_referer" "$http_user_agent"';
+    public static final Pattern COMBINED_PATTERN = Pattern.compile(
+            "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>.*)\".*$");
+
+    /**
+     * Instantiates a new http access log parser.
+     */
+    private HttpAccessLogParser()
+    {
+    }
+
+    /**
+     * Parses the log.
+     *
+     * @param line
+     *            the line
+     * @return the http log
+     */
+    public static HttpAccessLog parseLog(final String line, final Pattern pattern)
+    {
+        HttpAccessLog result;
+
+        Matcher matcher = pattern.matcher(line);
+        if (matcher.matches())
+        {
+            ZonedDateTime a = ZonedDateTime.now();
+            LocalDateTime b = a.toLocalDateTime();
+
+            result = new HttpAccessLog();
+            result.setIp(matcher.group("remoteAddress"));
+            result.setRemoteUser(matcher.group("remoteUser"));
+            result.setTime(ZonedDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
+            result.setRequest(matcher.group("request"));
+            result.setStatus(HttpStatusTable.instance().get(matcher.group("status")));
+            result.setBodyBytesSent(Long.valueOf(matcher.group("bodyBytesSent")));
+            result.setReferer(matcher.group("referer"));
+            result.setUserAgent(new UserAgent(matcher.group("userAgent")));
+        }
+        else
+        {
+            result = null;
+        }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.http;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.util.Files;
+
+/**
+ * The Class HttpAccessLogs.
+ */
+public class HttpAccessLogs implements Iterable<HttpAccessLog>
+{
+    private static Logger logger = LoggerFactory.getLogger(HttpAccessLogs.class);
+
+    private Files source;
+    private String regex;
+
+    /**
+     * Instantiates a new http access logs.
+     *
+     * @param source
+     *            the source
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public HttpAccessLogs(final Files source) throws IOException
+    {
+        this(source, null);
+    }
+
+    /**
+     * Instantiates a new http log iterator.
+     *
+     * @param source
+     *            the source
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public HttpAccessLogs(final Files source, final String regex) throws IOException
+    {
+        this.source = source;
+        this.regex = regex;
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Iterable#iterator()
+     */
+    @Override
+    public Iterator<HttpAccessLog> iterator()
+    {
+        HttpAccessLogIterator result;
+
+        try
+        {
+            result = new HttpAccessLogIterator(this.source, this.regex);
+        }
+        catch (IOException exception)
+        {
+            throw new IllegalArgumentException("Error with iterator.", exception);
+        }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -97,7 +97,7 @@ public class HttpLogIterator
         try
         {
             String line = this.iterator.next();
-            result = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
+            result = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
         }
         catch (Exception exception)
         {

src/fr/devinsy/statoolinfos/metrics/http/userAgentBotDetectorData.txt

@@ -1,4 +1,6 @@
+AhrefsBot
 ^Apache-HttpClient/
+ Barkrowler/
 bot
 BingPreview
 crawler
@@ -17,6 +19,7 @@ internal dummy connection
 ^Java/
 ^JGit/
  Let's Encrypt validation server;
+ltx71 
 ^MastoPeek 
 ^mattermost-
 ^Misskey/
@@ -24,6 +27,7 @@ internal dummy connection
 monitoring
 ^newspaper/
 ^node-fetch/
+Nutch-
 ^okhttp/
 ^PeerTube/
 ^PHP/
@@ -31,6 +35,9 @@ monitoring
 ^python-requests/
 ^python/
 ^Python/
+^Python-urllib/
+ SemrushBot
 ^Synapse/
 ^Tusky/
 YisouSpider
+ zgrab/

src/fr/devinsy/statoolinfos/metrics/libreqr/LibreQRProber.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.libreqr;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.metrics.UserCounters;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
+import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
+import fr.devinsy.statoolinfos.util.Files;
+
+/**
+ * The Class LibreQRProber.
+ */
+public class LibreQRProber
+{
+    private static Logger logger = LoggerFactory.getLogger(LibreQRProber.class);
+
+    /**
+     * Instantiates a new privatebin prober.
+     */
+    private LibreQRProber()
+    {
+    }
+
+    /**
+     * Probe.
+     *
+     * @param httpLogs
+     *            the http logs
+     * @param httpLogRegex
+     *            the http log regex
+     * @param dataPath
+     *            the data path
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
+    {
+        PathCounters result;
+
+        // metrics.service.users
+        // metrics.service.users.ipv4
+        // metrics.service.users.ipv6
+        // metrics.barcodes.count
+        // metrics.libreqr.barcodes.downloads
+        result = probeHttpAccessLog(httpLogs, httpLogRegex);
+
+        // metrics.service.files.bytes, metrics.libreqr.cache.bytes
+        // metrics.service.files.count, metrics.libreqr.cache.count
+        result.putAll(DatafilesProber.probe(dataPath));
+
+        //
+        return result;
+    }
+
+    /**
+     * Probe.
+     *
+     * @param httpAccessLogFiles
+     *            the http access log files
+     * @param httpRegex
+     *            the http regex
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    private static PathCounters probeHttpAccessLog(final Files httpAccessLogFiles, final String httpRegex) throws IOException, StatoolInfosException
+    {
+        PathCounters result;
+
+        result = new PathCounters();
+
+        //
+        UserCounters users = new UserCounters();
+        UserCounters ipv4Users = new UserCounters();
+        UserCounters ipv6Users = new UserCounters();
+
+        Pattern USE_PATTERN = Pattern.compile("GET /temp/\\w+\\.png.*");
+        Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
+
+        //
+        for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogFiles, httpRegex))
+        {
+            // General HTTP access logs.
+            String year = log.getYear();
+            String yearMonth = log.getYearMonth();
+            String yearWeek = log.getYearWeek();
+            String date = log.getDate();
+
+            // metrics.service.users
+            // metrics.service.users.ipv4
+            // metrics.service.users.ipv6
+            if ((!log.isBot()) && (USE_PATTERN.matcher(log.getRequest()).matches()))
+            {
+                String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
+
+                users.put(key, year, yearMonth, yearWeek, date);
+
+                if (log.isIPv4())
+                {
+                    ipv4Users.put(key, year, yearMonth, yearWeek, date);
+                }
+                else
+                {
+                    ipv6Users.put(key, year, yearMonth, yearWeek, date);
+                }
+            }
+
+            // metrics.barcodes.count
+            if ((log.getStatus().getCode() == 200) && (CREATE_PATTERN.matcher(log.getRequest()).matches()))
+            {
+                result.inc("metrics.barcodes.count", year, yearMonth, yearWeek, date);
+            }
+
+            // metrics.libreqr.barcodes.downloads
+            if (USE_PATTERN.matcher(log.getRequest()).matches())
+            {
+                result.inc("metrics.libreqr.barcodes.downloads", year, yearMonth, yearWeek, date);
+            }
+        }
+
+        //
+        result.putAll(users.getCounters("metrics.service.users"));
+        result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
+        result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinDataAnalyzer.java

@@ -1,91 +0,0 @@
-/*
- * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
- * 
- * This file is part of StatoolInfos, simple service statistics tool.
- * 
- * StatoolInfos is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- * 
- * StatoolInfos is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- * 
- * You should have received a copy of the GNU Affero General Public License
- * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
- */
-package fr.devinsy.statoolinfos.metrics.privatebin;
-
-import java.io.File;
-import java.io.IOException;
-import java.time.LocalDate;
-
-import org.apache.commons.io.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import fr.devinsy.statoolinfos.core.StatoolInfosException;
-import fr.devinsy.statoolinfos.metrics.PathCounters;
-import fr.devinsy.statoolinfos.metrics.TimeMark;
-import fr.devinsy.statoolinfos.util.FilesUtils;
-
-/**
- * The Class PrivatebinDataAnalyzer.
- */
-public class PrivatebinDataAnalyzer
-{
-    private static Logger logger = LoggerFactory.getLogger(PrivatebinDataAnalyzer.class);
-
-    /**
-     * Instantiates a new http access log prober.
-     */
-    private PrivatebinDataAnalyzer()
-    {
-    }
-
-    /**
-     * Probe.
-     *
-     * @param dataDirectory
-     *            the data directory
-     * @return the path counters
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
-     * @throws StatoolInfosException
-     *             the statool infos exception
-     */
-    public static PathCounters probe(final File dataDirectory) throws IOException, StatoolInfosException
-    {
-        PathCounters result;
-
-        System.out.println("Probing directory [" + dataDirectory + "]");
-
-        result = new PathCounters();
-
-        if ((dataDirectory.exists()) && (dataDirectory.isDirectory()))
-        {
-            LocalDate now = LocalDate.now();
-            String year = TimeMark.yearOf(now).toString();
-            String yearMonth = TimeMark.yearMonthOf(now).toString();
-            String yearWeek = TimeMark.yearWeekOf(now).toString();
-            String date = TimeMark.dayOf(now).toString();
-
-            // metrics.service.files.bytes
-            long size = FileUtils.sizeOfDirectory(dataDirectory);
-            result.set(size, "metrics.service.files.bytes", year, yearMonth, yearWeek, date);
-
-            // metrics.pastebins.count
-            long count = FilesUtils.searchByWildcard(dataDirectory.getAbsolutePath() + "/??/*").size();
-            result.set(count, "metrics.pastebins.count", year, yearMonth, yearWeek, date);
-        }
-        else
-        {
-            System.out.println("WARNING: Privatebin data path is not valid.");
-        }
-
-        //
-        return result;
-    }
-}

src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinHttpLogAnalyzer.java

@@ -1,215 +0,0 @@
-/*
- * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
- * 
- * This file is part of StatoolInfos, simple service statistics tool.
- * 
- * StatoolInfos is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- * 
- * StatoolInfos is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- * 
- * You should have received a copy of the GNU Affero General Public License
- * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
- */
-package fr.devinsy.statoolinfos.metrics.privatebin;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import fr.devinsy.statoolinfos.core.StatoolInfosException;
-import fr.devinsy.statoolinfos.metrics.PathCounters;
-import fr.devinsy.statoolinfos.metrics.UserCounters;
-import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
-import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer;
-import fr.devinsy.statoolinfos.util.FilesUtils;
-import fr.devinsy.statoolinfos.util.LineIterator;
-
-/**
- * The Class HttpAccessLogProber.
- */
-public class PrivatebinHttpLogAnalyzer
-{
-    private static Logger logger = LoggerFactory.getLogger(PrivatebinHttpLogAnalyzer.class);
-
-    private PathCounters counters;
-    private UserCounters users;
-    private UserCounters ipv4Users;
-    private UserCounters ipv6Users;
-
-    /**
-     * Instantiates a new http access log prober.
-     */
-    public PrivatebinHttpLogAnalyzer()
-    {
-        this.counters = new PathCounters();
-        this.users = new UserCounters();
-        this.ipv4Users = new UserCounters();
-        this.ipv6Users = new UserCounters();
-    }
-
-    /**
-     * Gets the counters.
-     *
-     * @return the counters
-     */
-    public PathCounters getCounters()
-    {
-        PathCounters result;
-
-        result = new PathCounters();
-        result.putAll(this.counters);
-
-        result.putAll(this.users.getCounters("metrics.service.users"));
-        result.putAll(this.ipv4Users.getCounters("metrics.service.users.ipv4"));
-        result.putAll(this.ipv6Users.getCounters("metrics.service.users.ipv6"));
-
-        //
-        return result;
-    }
-
-    /**
-     * Probe.
-     *
-     * @param file
-     *            the file
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
-     * @throws StatoolInfosException
-     *             the statool infos exception
-     */
-    public void probe(final File file, final String patternRegex) throws IOException
-    {
-        System.out.println("Probing file [" + file.getAbsolutePath() + "]");
-
-        //
-        Pattern pattern;
-        if (patternRegex == null)
-        {
-            pattern = HttpAccessLogAnalyzer.COMBINED_PATTERN;
-        }
-        else
-        {
-            pattern = Pattern.compile(patternRegex);
-        }
-
-        //
-        LineIterator iterator = new LineIterator(file);
-        while (iterator.hasNext())
-        {
-            String line = iterator.next();
-
-            try
-            {
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, pattern);
-                if (log == null)
-                {
-                    logger.warn("LINE IS NOT MATCHING [{}]", line);
-                }
-                else
-                {
-                    probeLog(log);
-                }
-            }
-            catch (Exception exception)
-            {
-                logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
-                exception.printStackTrace();
-            }
-        }
-    }
-
-    /**
-     * Probe log.
-     *
-     * @param log
-     *            the log
-     */
-    public void probeLog(final HttpAccessLog log)
-    {
-        if (log != null)
-        {
-            // logger.info("LINE IS MATCHING [{}]", log);
-            // logger.info(log.getHttpUserAgent().toString());
-
-            // General HTTP access logs.
-            String year = log.getYear();
-            String yearMonth = log.getYearMonth();
-            String yearWeek = log.getYearWeek();
-            String date = log.getDate();
-
-            // metrics.service.users
-            // metrics.service.users.ipv4
-            // metrics.service.users.ipv6
-            if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
-            {
-                String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
-                this.users.put(key, year, yearMonth, yearWeek, date);
-
-                if (log.isIPv4())
-                {
-                    this.ipv4Users.put(key, year, yearMonth, yearWeek, date);
-                }
-                else
-                {
-                    this.ipv6Users.put(key, year, yearMonth, yearWeek, date);
-                }
-            }
-
-            // metrics.pastebins.created
-            // metrics.pastebins.read
-            // metrics.pastebins.deleted
-            if (StringUtils.startsWith(log.getRequest(), "POST "))
-            {
-                this.counters.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
-            }
-            else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
-            {
-                if (StringUtils.contains(log.getRequest(), "&deletetoken="))
-                {
-                    this.counters.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
-                }
-                else
-                {
-                    this.counters.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
-                }
-            }
-        }
-    }
-
-    /**
-     * Probe.
-     *
-     * @param httpAccessLogs
-     *            the source
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
-     * @throws StatoolInfosException
-     *             the statool infos exception
-     */
-    public static PathCounters probe(final String httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
-    {
-        PathCounters result;
-
-        PrivatebinHttpLogAnalyzer analyzer = new PrivatebinHttpLogAnalyzer();
-
-        for (File file : FilesUtils.searchByWildcard(httpAccessLogs))
-        {
-            analyzer.probe(file, httpRegex);
-        }
-
-        result = analyzer.getCounters();
-
-        //
-        return result;
-    }
-}

src/fr/devinsy/statoolinfos/metrics/privatebin/PrivatebinProber.java

@@ -21,11 +21,18 @@ package fr.devinsy.statoolinfos.metrics.privatebin;
 import java.io.File;
 import java.io.IOException;
 
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import fr.devinsy.statoolinfos.core.StatoolInfosException;
 import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.metrics.UserCounters;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogs;
+import fr.devinsy.statoolinfos.metrics.util.DatafilesProber;
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.statoolinfos.util.FilesUtils;
 
 /**
  * The Class PrivatebinProber.
@@ -37,10 +44,80 @@ public class PrivatebinProber
     /**
      * Instantiates a new privatebin prober.
      */
-    public PrivatebinProber()
+    private PrivatebinProber()
     {
     }
 
+    private static PathCounters probe(final Files httpAccessLogs, final String httpRegex) throws IOException, StatoolInfosException
+    {
+        PathCounters result;
+
+        result = new PathCounters();
+
+        //
+        UserCounters users = new UserCounters();
+        UserCounters ipv4Users = new UserCounters();
+        UserCounters ipv6Users = new UserCounters();
+
+        //
+        for (HttpAccessLog log : new HttpAccessLogs(httpAccessLogs, httpRegex))
+        {
+            // logger.info("LINE IS MATCHING [{}]", log);
+            // logger.info(log.getHttpUserAgent().toString());
+
+            // General HTTP access logs.
+            String year = log.getYear();
+            String yearMonth = log.getYearMonth();
+            String yearWeek = log.getYearWeek();
+            String date = log.getDate();
+
+            // metrics.service.users
+            // metrics.service.users.ipv4
+            // metrics.service.users.ipv6
+            if (StringUtils.startsWithAny(log.getRequest(), "POST ", "GET /?pasteid="))
+            {
+                String key = String.format("%s---%s", log.getIp(), log.getUserAgent());
+                users.put(key, year, yearMonth, yearWeek, date);
+
+                if (log.isIPv4())
+                {
+                    ipv4Users.put(key, year, yearMonth, yearWeek, date);
+                }
+                else
+                {
+                    ipv6Users.put(key, year, yearMonth, yearWeek, date);
+                }
+            }
+
+            // metrics.pastebins.created
+            // metrics.pastebins.read
+            // metrics.pastebins.deleted
+            if (StringUtils.startsWith(log.getRequest(), "POST "))
+            {
+                result.inc("metrics.pastebins.created", year, yearMonth, yearWeek, date);
+            }
+            else if (StringUtils.startsWith(log.getRequest(), "GET /?pasteid="))
+            {
+                if (StringUtils.contains(log.getRequest(), "&deletetoken="))
+                {
+                    result.inc("metrics.pastebins.deleted", year, yearMonth, yearWeek, date);
+                }
+                else
+                {
+                    result.inc("metrics.pastebins.read", year, yearMonth, yearWeek, date);
+                }
+            }
+        }
+
+        //
+        result.putAll(users.getCounters("metrics.service.users"));
+        result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
+        result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
+
+        //
+        return result;
+    }
+
     /**
      * Probe.
      *
@@ -48,7 +125,7 @@ public class PrivatebinProber
      *            the http logs
      * @param httpLogRegex
      *            the http log regex
-     * @param dataPath
+     * @param datafileDirectory
      *            the data path
      * @return the path counters
      * @throws IOException
@@ -56,7 +133,7 @@ public class PrivatebinProber
      * @throws StatoolInfosException
      *             the statool infos exception
      */
-    public static PathCounters probe(final String httpLogs, final String httpLogRegex, final File dataPath) throws IOException, StatoolInfosException
+    public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final File datafileDirectory) throws IOException, StatoolInfosException
     {
         PathCounters result;
 
@@ -66,11 +143,18 @@ public class PrivatebinProber
         // metrics.pastebins.created
         // metrics.pastebins.read
         // metrics.pastebins.deleted
-        result = PrivatebinHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
+        result = probe(httpLogs, httpLogRegex);
 
         // metrics.service.files.bytes
+        // metrics.service.files.count
+        result.putAll(DatafilesProber.probe(datafileDirectory));
+
         // metrics.pastebins.count
-        result.putAll(PrivatebinDataAnalyzer.probe(dataPath));
+        if ((datafileDirectory != null) && (datafileDirectory.exists()) && (datafileDirectory.isDirectory()))
+        {
+            long count = FilesUtils.searchByWildcard(datafileDirectory.getAbsolutePath() + "/??/*").size();
+            result.set(count, "metrics.pastebins.count");
+        }
 
         // metrics.pastebins.purged
         // purged = count(n-1) - count(n) + created - delete

src/fr/devinsy/statoolinfos/metrics/util/DatabaseProber.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.util;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.core.DatabaseConfig;
+import fr.devinsy.statoolinfos.metrics.PathCounters;
+
+/**
+ * The Class DatabaseProber.
+ */
+public class DatabaseProber
+{
+    private static Logger logger = LoggerFactory.getLogger(DatabaseProber.class);
+
+    /**
+     * Instantiates a new database prober.
+     */
+    private DatabaseProber()
+    {
+    }
+
+    /**
+     * Probe.
+     *
+     * @param config
+     *            the config
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public static PathCounters probe(final DatabaseConfig config) throws IOException
+    {
+        PathCounters result;
+
+        result = new PathCounters();
+
+        if (config != null)
+        {
+            System.out.println("Probing directory [" + config.getName() + "]");
+        }
+
+        // if ((directory != null) && (directory.exists()) &&
+        // (directory.isDirectory()))
+        // {
+        // StringList timemarks = result.getNowTimeMarks();
+        //
+        // // metrics.service.database.bytes
+        // long size = FileUtils.sizeOfDirectory(directory);
+        // result.set(size, prefix + ".files.bytes", timemarks);
+        // }
+        // else
+        // {
+        // System.out.println("WARNING: datafile path not valid.");
+        // }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/util/DatafilesProber.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.util;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.util.FilesUtils;
+import fr.devinsy.strings.StringList;
+
+/**
+ * The Class DatafilesProber.
+ */
+public class DatafilesProber
+{
+    private static Logger logger = LoggerFactory.getLogger(DatafilesProber.class);
+
+    /**
+     * Instantiates a new datafiles prober.
+     */
+    private DatafilesProber()
+    {
+    }
+
+    /**
+     * Probe.
+     *
+     * @param directory
+     *            the directory
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    public static PathCounters probe(final File directory) throws IOException
+    {
+        PathCounters result;
+
+        result = probe(directory, "metrics.service");
+
+        //
+        return result;
+    }
+
+    /**
+     * Probe.
+     *
+     * @param directory
+     *            the data directory
+     * @return the path counters
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     * @throws StatoolInfosException
+     *             the statool infos exception
+     */
+    public static PathCounters probe(final File directory, final String prefix) throws IOException
+    {
+        PathCounters result;
+
+        System.out.println("Probing directory [" + directory + "]");
+
+        result = new PathCounters();
+
+        if ((directory != null) && (directory.exists()) && (directory.isDirectory()))
+        {
+            StringList timemarks = result.getNowTimeMarks();
+
+            // metrics.service.datafiles.bytes
+            long size = FileUtils.sizeOfDirectory(directory);
+            result.set(size, prefix + ".datafiles.bytes", timemarks);
+
+            // metrics.service.datafiles.count
+            long count = FilesUtils.listRecursively(directory.getAbsoluteFile()).size();
+            result.set(count, prefix + ".datafiles.count", timemarks);
+        }
+        else
+        {
+            System.out.println("WARNING: datafile path not valid.");
+        }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/util/Files.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -49,6 +49,56 @@ public class Files extends ArrayList<File>
         super(initialCapacity);
     }
 
+    /**
+     * Keep directories.
+     *
+     * @return the files
+     */
+    public Files keepDirectoryType()
+    {
+        Files result;
+
+        Iterator<File> iterator = iterator();
+        while (iterator.hasNext())
+        {
+            File file = iterator.next();
+            if (!file.isDirectory())
+            {
+                iterator.remove();
+            }
+        }
+
+        result = this;
+
+        //
+        return result;
+    }
+
+    /**
+     * Keep file type.
+     *
+     * @return the files
+     */
+    public Files keepFileType()
+    {
+        Files result;
+
+        Iterator<File> iterator = iterator();
+        while (iterator.hasNext())
+        {
+            File file = iterator.next();
+            if (!file.isFile())
+            {
+                iterator.remove();
+            }
+        }
+
+        result = this;
+
+        //
+        return result;
+    }
+
     /**
      * Keep.
      *
@@ -96,7 +146,6 @@ public class Files extends ArrayList<File>
         while (iterator.hasNext())
         {
             File file = iterator.next();
-
             if (!pattern.matcher(file.getAbsolutePath()).matches())
             {
                 iterator.remove();
@@ -109,56 +158,6 @@ public class Files extends ArrayList<File>
         return result;
     }
 
-    /**
-     * Keep directories.
-     *
-     * @return the files
-     */
-    public Files keepDirectoryType()
-    {
-        Files result;
-
-        Iterator<File> iterator = iterator();
-        while (iterator.hasNext())
-        {
-            File file = iterator.next();
-            if (!file.isDirectory())
-            {
-                iterator.remove();
-            }
-        }
-
-        result = this;
-
-        //
-        return result;
-    }
-
-    /**
-     * Keep file type.
-     *
-     * @return the files
-     */
-    public Files keepFileType()
-    {
-        Files result;
-
-        Iterator<File> iterator = iterator();
-        while (iterator.hasNext())
-        {
-            File file = iterator.next();
-            if (!file.isFile())
-            {
-                iterator.remove();
-            }
-        }
-
-        result = this;
-
-        //
-        return result;
-    }
-
     /**
      * Removes the containing.
      *

src/fr/devinsy/statoolinfos/util/FilesLineIterator.java

@@ -0,0 +1,181 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The Class WildcardFileLineIterator.
+ */
+public class FilesLineIterator implements Iterator<String>
+{
+    private static Logger logger = LoggerFactory.getLogger(FilesLineIterator.class);
+
+    private Iterator<File> fileIterator;
+    private LineIterator lineIterator;
+    private boolean print;
+
+    /**
+     * Instantiates a new wildcard file line iterator.
+     *
+     * @param source
+     *            the source
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public FilesLineIterator(final Files source)
+    {
+        this.fileIterator = source.iterator();
+        this.lineIterator = null;
+        setPrintOn();
+    }
+
+    /**
+     * Forward.
+     */
+    private void forward()
+    {
+        boolean ended = false;
+        while (!ended)
+        {
+            try
+            {
+                if (this.lineIterator == null)
+                {
+                    if (this.fileIterator.hasNext())
+                    {
+                        File file = this.fileIterator.next();
+                        if (this.print)
+                        {
+                            System.out.println("Iterating file [" + file.getAbsolutePath() + "]");
+                        }
+                        this.lineIterator = new LineIterator(file);
+                    }
+                    else
+                    {
+                        ended = true;
+                    }
+                }
+                else if (this.lineIterator.hasNext())
+                {
+                    ended = true;
+                }
+                else
+                {
+                    this.lineIterator.close();
+                    if (this.fileIterator.hasNext())
+                    {
+                        File file = this.fileIterator.next();
+                        if (this.print)
+                        {
+                            System.out.println("Iterating file [" + file.getAbsolutePath() + "]");
+                        }
+                        this.lineIterator = new LineIterator(file);
+                    }
+                    else
+                    {
+                        this.lineIterator = null;
+                        ended = true;
+                    }
+                }
+            }
+            catch (IOException exception)
+            {
+                logger.error("ERROR READING.", exception);
+                if (this.lineIterator != null)
+                {
+                    this.lineIterator.close();
+                    this.lineIterator = null;
+                }
+            }
+        }
+    }
+
+    /* (non-Javadoc)
+     * @see java.util.Iterator#hasNext()
+     */
+    @Override
+    public boolean hasNext()
+    {
+        boolean result;
+
+        forward();
+
+        if ((this.lineIterator == null) && (!this.fileIterator.hasNext()))
+        {
+            result = false;
+        }
+        else
+        {
+            result = true;
+        }
+
+        //
+        return result;
+    }
+
+    /* (non-Javadoc)
+     * @see java.util.Iterator#next()
+     */
+    @Override
+    public String next()
+    {
+        String result;
+
+        try
+        {
+
+            forward();
+
+            if (this.lineIterator == null)
+            {
+                result = null;
+            }
+            else
+            {
+                result = this.lineIterator.next();
+            }
+        }
+        catch (IOException exception)
+        {
+            throw new IllegalArgumentException("ERROR READING.", exception);
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Sets the print off.
+     */
+    public void setPrintOff()
+    {
+        this.print = false;
+    }
+
+    public void setPrintOn()
+    {
+        this.print = true;
+    }
+}

src/fr/devinsy/statoolinfos/util/FilesUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -121,7 +121,9 @@ public class FilesUtils
         {
             // Get the first parent without wildcard.
             File parent = null;
-            File current = new File(source);
+            File path = new File(source).getAbsoluteFile();
+            File current = path;
+
             boolean ended = false;
             while (!ended)
             {
@@ -146,7 +148,7 @@ public class FilesUtils
 
             //
             System.out.println("parent=" + parent);
-            String regex = source.replace(".", "\\.").replace("?", ".").replace("*", ".*");
+            String regex = path.toString().replace(".", "\\.").replace("?", ".").replace("*", ".*");
             result = listRecursively(parent).keepPath(regex);
         }