Extended Etherpad probing.

README.md

@@ -126,7 +126,23 @@ Create a cron file to update the metric file everyday:
 
 Warning: in previous day mode, the metrics generated are overwrited for the last month, the last week and the last day. So, six weeks in logs are required.
 
+### Etherpad metrics
 
+Configuration template:
+
+```
+conf.probe.types=Etherpad
+conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
+conf.probe.httpaccesslog.pattern=
+# jdbc:mariadb://localhost:1234/databasename
+# jdbc:mysql://localhost:1234/databasename
+# jdbc:postgresql://localhost:1234/databasename
+# jdbc:sqlite:/foo/bar/databasename.sqlite
+conf.probe.gitea.database.url=
+conf.probe.gitea.database.user=
+conf.probe.gitea.database.password=
+conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
+```
 
 ### Framdadate metrics (coming soon)
 

src/fr/devinsy/statoolinfos/metrics/Prober.java

@@ -140,12 +140,14 @@ public class Prober
                 logger.info("== Probing Etherpad.");
                 String httpLogs = configuration.getProbeHttpAccessLogSource();
                 logger.info("httpLogs=[{}]", httpLogs);
+                String logs = configuration.get("conf.probe.etherpad.logs");
+                logger.info("logs=[{}]", logs);
                 String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
                 logger.info("httpAccessPattern=[{}]", httpAccessLogRegex);
-                DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.gitea");
+                DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.etherpad");
                 logger.info("database={}", database.toString());
 
-                PathCounters metrics = EtherpadProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, database);
+                PathCounters metrics = EtherpadProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, FilesUtils.searchByWildcard(logs), database);
                 counters.putAll(metrics);
             }
 

src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java

@@ -38,8 +38,7 @@ public class EtherpadHttpLogAnalyzer
 {
     private static Logger logger = LoggerFactory.getLogger(EtherpadHttpLogAnalyzer.class);
 
-    public static final Pattern USE_PATTERN = Pattern.compile("GET /p/\\S+ ");
+    public static final Pattern USE_PATTERN = Pattern.compile("^GET /p/\\S+ .*$");
-    public static final Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
 
     private PathCounters counters;
     private UserCounters users;

src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLog.java

@@ -0,0 +1,291 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.etherpad;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.metrics.TimeMark;
+import fr.devinsy.statoolinfos.metrics.TimeMarkUtils;
+import fr.devinsy.strings.StringList;
+
+/**
+ * The Class EtherpadLog.
+ */
+public class EtherpadLog
+{
+    private static Logger logger = LoggerFactory.getLogger(EtherpadLog.class);
+
+    private LocalDateTime time;
+    private String level;
+    private String type;
+    private String event;
+    private String padname;
+    private String ip;
+    private String author;
+
+    /**
+     * Instantiates a new etherpad log.
+     */
+    public EtherpadLog()
+    {
+        this.time = null;
+        this.level = null;
+        this.type = null;
+        this.event = null;
+        this.padname = null;
+        this.ip = null;
+        this.author = null;
+    }
+
+    public String getAuthor()
+    {
+        return this.author;
+    }
+
+    /**
+     * Gets the date.
+     *
+     * @return the date
+     */
+    public String getDate()
+    {
+        String result;
+
+        result = this.time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
+
+        //
+        return result;
+    }
+
+    public String getEvent()
+    {
+        return this.event;
+    }
+
+    public String getIp()
+    {
+        return this.ip;
+    }
+
+    public String getLevel()
+    {
+        return this.level;
+    }
+
+    public String getPadname()
+    {
+        return this.padname;
+    }
+
+    public LocalDateTime getTime()
+    {
+        return this.time;
+    }
+
+    /**
+     * Gets the time marks.
+     *
+     * @return the time marks
+     */
+    public StringList getTimeMarks()
+    {
+        StringList result;
+
+        result = new StringList();
+
+        String year = TimeMark.yearOf(this.time).toString();
+        String yearMonth = TimeMark.yearMonthOf(this.time).toString();
+        String yearWeek = TimeMark.yearWeekOf(this.time).toString();
+        String date = TimeMark.dayOf(this.time).toString();
+
+        result.add(year);
+        result.add(yearMonth);
+        result.add(yearWeek);
+        result.add(date);
+
+        //
+        return result;
+    }
+
+    public String getType()
+    {
+        return this.type;
+    }
+
+    /**
+     * Gets the year.
+     *
+     * @return the year
+     */
+    public String getYear()
+    {
+        String result;
+
+        if (this.time == null)
+        {
+            result = null;
+        }
+        else
+        {
+            result = TimeMarkUtils.yearOf(this.time);
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the year month.
+     *
+     * @return the year month
+     */
+    public String getYearMonth()
+    {
+        String result;
+
+        if (this.time == null)
+        {
+            result = null;
+        }
+        else
+        {
+            result = TimeMarkUtils.yearMonthOf(this.time);
+        }
+        //
+        return result;
+    }
+
+    /**
+     * Gets the year week.
+     *
+     * @return the year week
+     */
+    public String getYearWeek()
+    {
+        String result;
+        if (this.time == null)
+        {
+            result = null;
+        }
+        else
+        {
+            result = TimeMarkUtils.yearWeekOf(this.time);
+        }
+        //
+        return result;
+    }
+
+    /**
+     * Checks if is ipv4.
+     *
+     * @return true, if is ipv4
+     */
+    public boolean isIPv4()
+    {
+        boolean result;
+
+        if (this.ip == null)
+        {
+            result = false;
+        }
+        else
+        {
+            result = this.ip.contains(".");
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Checks if is ipv6.
+     *
+     * @return true, if is ipv6
+     */
+    public boolean isIPv6()
+    {
+        boolean result;
+
+        if (this.ip == null)
+        {
+            result = false;
+        }
+        else
+        {
+            result = this.ip.contains(":");
+        }
+
+        //
+        return result;
+    }
+
+    public void setAuthor(final String author)
+    {
+        this.author = author;
+    }
+
+    public void setEvent(final String event)
+    {
+        this.event = event;
+    }
+
+    public void setIp(final String ip)
+    {
+        this.ip = ip;
+    }
+
+    public void setLevel(final String level)
+    {
+        this.level = level;
+    }
+
+    public void setPadname(final String padname)
+    {
+        this.padname = padname;
+    }
+
+    public void setTime(final LocalDateTime time)
+    {
+        this.time = time;
+    }
+
+    public void setType(final String type)
+    {
+        this.type = type;
+    }
+
+    /* (non-Javadoc)
+     * @see java.lang.Object#toString()
+     */
+    @Override
+    public String toString()
+    {
+        String result;
+
+        result = String.format("[time=%s, level=%s, type=%s, event=%s, %padname=%s, ip=%s, auhtor=%s]", this.time, this.level, this.type, this.event, this.padname, this.ip, this.author);
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLogProber.java

@@ -0,0 +1,224 @@
+/*
+ * Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.etherpad;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.metrics.UserCounters;
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.statoolinfos.util.LineIterator;
+
+/**
+ * The Class EtherpadLogProber.
+ */
+public class EtherpadLogProber
+{
+    private static Logger logger = LoggerFactory.getLogger(EtherpadLogProber.class);
+
+    // 2021-11-17 01:51:59 DELETE 7922dd508e0cf9b2
+    public static final Pattern LOG_PATTERN = Pattern.compile(
+            "^\\[(?<datetime>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(\\.\\d{3})?)]\\s\\[(?<level>[^]]+)]\\s+(?<type>\\S+) - \\[(?<event>[^]]+)]\\spad:(?<padname>\\S+)\\ssocket:\\S+\\sIP:(?<ip>\\S+) authorID:(?<author>\\S+)( username:.+)?$");
+    /*
+    [2022-03-28 02:19:35.842] [INFO] access - [ENTER] pad:test socket:Qr8BEX3mP6EmheyLAAAA IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
+    [2022-03-28 18:50:10.963] [INFO] access - [CREATE] pad:test7 socket:WddndPw_4vt01-0YAAAC IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
+    [2022-03-28 18:52:00.668] [INFO] access - [ENTER] pad:test7 socket:gA4Ba57988zrGJNmAAAD IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
+    [2022-03-28 18:52:27.947] [INFO] access - [ENTER] pad:test7 socket:9ZrFheETmDUpUcS1AAAE IP:82.65.227.202 authorID:a.AMpqExl3JHbPL52c
+    [2022-03-28 18:59:58.501] [INFO] access - [ENTER] pad:test socket:mCgPMxg2wb7upHP8AAAG IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
+    [2022-03-28 19:01:24.884] [INFO] access - [LEAVE] pad:test socket:mCgPMxg2wb7upHP8AAAG IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
+     */
+
+    public static final Pattern DELETE_EMPTY_LOG_PATTERN = Pattern.compile(
+            "^\\[(?<datetime>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(\\.\\d{3})?)]\\s\\[(?<level>[^]]+)]\\s+(?<type>\\S+) - Deleting (?<padname>\\S+) when user leaved since empty$");
+    /*
+     [2022-03-29 00:43:57.085] [INFO] ep_delete_empty_pads - Deleting test8 when user leaved since empty
+     */
+
+    /**
+     * Instantiates a new etherpad log prober.
+     */
+    private EtherpadLogProber()
+    {
+    }
+
+    /**
+     * Parses the log.
+     *
+     * @param line
+     *            the line
+     * @return the http log
+     */
+    public static EtherpadLog parseLog(final String line)
+    {
+        EtherpadLog result;
+
+        Matcher matcher = LOG_PATTERN.matcher(line);
+        if (matcher.matches())
+        {
+            result = new EtherpadLog();
+            result.setTime(LocalDateTime.parse(matcher.group("datetime"), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withLocale(Locale.ENGLISH)));
+            result.setLevel(matcher.group("level"));
+            result.setType(matcher.group("type"));
+            result.setEvent(matcher.group("event"));
+            result.setPadname(matcher.group("padname"));
+            result.setIp(matcher.group("ip"));
+            result.setAuthor(matcher.group("author"));
+        }
+        else
+        {
+            matcher = DELETE_EMPTY_LOG_PATTERN.matcher(line);
+            if (matcher.matches())
+            {
+                result = new EtherpadLog();
+                result.setTime(LocalDateTime.parse(matcher.group("datetime"), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withLocale(Locale.ENGLISH)));
+                result.setLevel(matcher.group("level"));
+                result.setType(matcher.group("type"));
+                result.setEvent("Deleting");
+                result.setAuthor(matcher.group("padname"));
+            }
+            else
+            {
+                result = null;
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Probe.
+     *
+     * @param file
+     *            the file
+     * @return the path counters
+     */
+    public static PathCounters probe(final Files files) throws IOException
+    {
+        PathCounters result;
+
+        result = new PathCounters();
+
+        UserCounters users = new UserCounters();
+        UserCounters ipv4Users = new UserCounters();
+        UserCounters ipv6Users = new UserCounters();
+        UserCounters actives = new UserCounters();
+
+        for (File file : files)
+        {
+            System.out.println("Probing file [" + file.getAbsolutePath() + "]");
+
+            //
+            int lineCount = 0;
+            LineIterator iterator = new LineIterator(file);
+            while (iterator.hasNext())
+            {
+                lineCount += 1;
+                String line = iterator.next();
+
+                // Remove ANSI Escape Codes.
+                line = line.replaceAll("\u001B\\[[\\d;]*[^\\d;]", "");
+
+                try
+                {
+                    EtherpadLog log = parseLog(line);
+                    // System.out.println("====> " + log);
+                    if (log == null)
+                    {
+                        if (line.contains("access -"))
+                        {
+                            logger.warn("LINE IS NOT MATCHING [{}]", line);
+                        }
+                    }
+                    else
+                    {
+                        // Timemarks.
+                        String year = log.getYear();
+                        String yearMonth = log.getYearMonth();
+                        String yearWeek = log.getYearWeek();
+                        String date = log.getDate();
+
+                        if (StringUtils.equals(log.getType(), "access"))
+                        {
+                            // metrics.service.users
+                            // metrics.service.users.ipv4
+                            // metrics.service.users.ipv6
+                            users.put(log.getAuthor(), year, yearMonth, yearWeek, date);
+                            if (log.isIPv4())
+                            {
+                                ipv4Users.put(log.getAuthor(), year, yearMonth, yearWeek, date);
+                            }
+                            else
+                            {
+                                ipv6Users.put(log.getAuthor(), year, yearMonth, yearWeek, date);
+                            }
+
+                            // metrics.etherpad.pads.actives
+                            actives.put(log.getPadname(), year, yearMonth, yearWeek, date);
+
+                            // metrics.etherpad.pads.created
+                            // alias metrics.textprocessors.created
+                            if (StringUtils.equals(log.getEvent(), "CREATE"))
+                            {
+                                result.inc("metrics.etherpad.pads.created", year, yearMonth, yearWeek, date);
+                                result.inc("metrics.textprocessors.created", year, yearMonth, yearWeek, date);
+                            }
+
+                            if (StringUtils.equalsAny(log.getEvent(), "CREATE", "ENTER"))
+                            {
+                                result.inc("metrics.etherpad.pads.reads", year, yearMonth, yearWeek, date);
+                                result.inc("metrics.textprocessors.reads", year, yearMonth, yearWeek, date);
+                            }
+                        }
+                        else if (StringUtils.equals(log.getType(), "ep_delete_empty_pads"))
+                        {
+                            result.inc("metrics.etherpad.pads.deleted", year, yearMonth, yearWeek, date);
+                            result.inc("metrics.textprocessors.deleted", year, yearMonth, yearWeek, date);
+                        }
+                    }
+                }
+                catch (Exception exception)
+                {
+                    logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
+                    exception.printStackTrace();
+                }
+            }
+            System.out.println("PrivatebinPatchLog line count=" + lineCount);
+        }
+
+        //
+        result.putAll(users.getCounters("metrics.service.users"));
+        result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
+        result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
+        result.putAll(actives.getCounters("metrics.etherpad.pads.actives"));
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java

@@ -61,14 +61,21 @@ public class EtherpadProber
      * @throws StatoolInfosException
      *             the statool infos exception
      */
-    public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException
+    public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final Files logs, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException
     {
         PathCounters result;
 
         // metrics.service.users
         // metrics.service.users.ipv4
         // metrics.service.users.ipv6
+        if (logs.isEmpty())
+        {
             result = EtherpadHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
+        }
+        else
+        {
+            result = EtherpadLogProber.probe(logs);
+        }
 
         try
         {
@@ -144,10 +151,6 @@ public class EtherpadProber
             logger.error("ERROR with database.", exception);
         }
 
-        /*
-         * select substring(key, strpos(key, ':')+1, strpos(substring(key, strpos(key, ':')+1), ':'))  from store where key like 'pad:%:revs:%';
-         */
-
         //
         return result;
     }