From 4ee90a3671d96ea2250760dc062c182fc54a07a7 Mon Sep 17 00:00:00 2001
From: "Christian P. MOMON" <christian.momon@devinsy.fr>
Date: Sat, 20 Feb 2021 02:00:00 +0100
Subject: [PATCH] Added http error log analyze.

---
 .../devinsy/statoolinfos/metrics/Prober.java  |   6 +-
 .../http/{HttpLog.java => HttpAccessLog.java} |  10 +-
 .../metrics/http/HttpAccessLogAnalyzer.java   |  55 +++---
 .../metrics/http/HttpErrorLog.java            | 163 ++++++++++++++++++
 .../metrics/http/HttpErrorLogAnalyzer.java    |  91 +++++++++-
 .../metrics/http/HttpLogIterator.java         |   4 +-
 .../metrics/http/VisitCounters.java           |   4 +-
 .../metrics/http/VisitCountersTest.java       |  24 +--
 8 files changed, 294 insertions(+), 63 deletions(-)
 rename src/fr/devinsy/statoolinfos/metrics/http/{HttpLog.java => HttpAccessLog.java} (96%)
 create mode 100644 src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLog.java

diff --git a/src/fr/devinsy/statoolinfos/metrics/Prober.java b/src/fr/devinsy/statoolinfos/metrics/Prober.java
index 36452c7..78b7741 100644
--- a/src/fr/devinsy/statoolinfos/metrics/Prober.java
+++ b/src/fr/devinsy/statoolinfos/metrics/Prober.java
@@ -114,11 +114,7 @@ public class Prober
         }
 
         //
-        {
-            PathCounter counter = counters.get("metrics.http.hits", "2020");
-            logger.info("counter=" + counter.getCounter());
-            logger.info("size={}", counters.size());
-        }
+        logger.info("size={}", counters.size());
     }
 
     /**
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpLog.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java
similarity index 96%
rename from src/fr/devinsy/statoolinfos/metrics/http/HttpLog.java
rename to src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java
index ea738c0..390b750 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/HttpLog.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java
@@ -31,11 +31,11 @@ import fr.devinsy.strings.StringList;
 import fr.devinsy.strings.StringsUtils;
 
 /**
- * The Class HttpLog.
+ * The Class HttpAccessLog.
  */
-public class HttpLog
+public class HttpAccessLog
 {
-    private static Logger logger = LoggerFactory.getLogger(HttpLog.class);
+    private static Logger logger = LoggerFactory.getLogger(HttpAccessLog.class);
 
     public static final Pattern IPV4_PATTERN = Pattern.compile("\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}");
     public static final Pattern IPV6_PATTERN = Pattern.compile("([0-9a-f]{1,4}:{1,2}){4,7}([0-9a-f]){1,4}", Pattern.CASE_INSENSITIVE);
@@ -50,9 +50,9 @@ public class HttpLog
     private UserAgent userAgent;
 
     /**
-     * Instantiates a new http log.
+     * Instantiates a new http access log.
      */
-    public HttpLog()
+    public HttpAccessLog()
     {
         this.remoteAddress = null;
         this.remoteUser = null;
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java
index 0f801a3..396bb8e 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java
@@ -124,8 +124,7 @@ public class HttpAccessLogAnalyzer
      */
     public void probeLine(final String line)
     {
-
-        HttpLog log = parseNginxCombinedLog(line);
+        HttpAccessLog log = parseNginxCombinedLog(line);
 
         // logger.info("==================");
         if (log == null)
@@ -171,7 +170,7 @@ public class HttpAccessLogAnalyzer
             }
 
             // metrics.http.pages.* =
-            if (StringUtils.endsWithAny(log.getRequest(), ".html", ".htm", ".xhtml", ".cgi"))
+            if (StringUtils.endsWithAny(log.getRequest(), ".html", ".htm", ".xhtml", ".cgi", "/"))
             {
                 this.counters.inc("metrics.http.pages", year, yearMonth, yearWeek, date);
             }
@@ -223,38 +222,16 @@ public class HttpAccessLogAnalyzer
      *            the line
      * @return the http log
      */
-    public static HttpLog parseNginxCombinedLog(final String line)
+    public static HttpAccessLog parseLog(final String line, final String pattern)
     {
-        HttpLog result;
-
-        // log_format combined '$remote_addr - $remote_user [$time_local] '
-        // '"$request" $status $body_bytes_sent '
-        // '"$http_referer" "$http_user_agent"';
-        String combinedPattern = "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$";
-
-        result = parseLog(line, combinedPattern);
-
-        //
-        return result;
-    }
-
-    /**
-     * Parses the log.
-     *
-     * @param line
-     *            the line
-     * @return the http log
-     */
-    public static HttpLog parseLog(final String line, final String pattern)
-    {
-        HttpLog result;
+        HttpAccessLog result;
 
         Pattern combined = Pattern.compile(pattern);
 
         Matcher matcher = combined.matcher(line);
         if (matcher.matches())
         {
-            result = new HttpLog();
+            result = new HttpAccessLog();
             result.setRemoteAddress(matcher.group("remoteAddress"));
             result.setRemoteUser(matcher.group("remoteUser"));
             result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
@@ -273,6 +250,28 @@ public class HttpAccessLogAnalyzer
         return result;
     }
 
+    /**
+     * Parses the log.
+     *
+     * @param line
+     *            the line
+     * @return the http log
+     */
+    public static HttpAccessLog parseNginxCombinedLog(final String line)
+    {
+        HttpAccessLog result;
+
+        // log_format combined '$remote_addr - $remote_user [$time_local] '
+        // '"$request" $status $body_bytes_sent '
+        // '"$http_referer" "$http_user_agent"';
+        String combinedPattern = "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$";
+
+        result = parseLog(line, combinedPattern);
+
+        //
+        return result;
+    }
+
     /**
      * Probe.
      *
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLog.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLog.java
new file mode 100644
index 0000000..07511f9
--- /dev/null
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLog.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.http;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.statoolinfos.metrics.TimeMarkUtils;
+import fr.devinsy.strings.StringList;
+
+/**
+ * The Class HttpErrorLog.
+ */
+public class HttpErrorLog
+{
+    private static Logger logger = LoggerFactory.getLogger(HttpErrorLog.class);
+
+    private LocalDateTime time;
+    private String level;
+
+    /**
+     * Instantiates a new http error log.
+     */
+    public HttpErrorLog()
+    {
+        this.time = null;
+        this.level = null;
+    }
+
+    public String getDate()
+    {
+        String result;
+
+        result = this.time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
+
+        //
+        return result;
+    }
+
+    public String getLevel()
+    {
+        return this.level;
+    }
+
+    public LocalDateTime getTime()
+    {
+        return this.time;
+    }
+
+    /**
+     * Gets the year.
+     *
+     * @return the year
+     */
+    public String getYear()
+    {
+        String result;
+
+        if (this.time == null)
+        {
+            result = null;
+        }
+        else
+        {
+            result = TimeMarkUtils.yearOf(this.time);
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Gets the year month.
+     *
+     * @return the year month
+     */
+    public String getYearMonth()
+    {
+        String result;
+
+        if (this.time == null)
+        {
+            result = null;
+        }
+        else
+        {
+            result = TimeMarkUtils.yearMonthOf(this.time);
+        }
+        //
+        return result;
+    }
+
+    /**
+     * Gets the year week.
+     *
+     * @return the year week
+     */
+    public String getYearWeek()
+    {
+        String result;
+        if (this.time == null)
+        {
+            result = null;
+        }
+        else
+        {
+            result = TimeMarkUtils.yearWeekOf(this.time);
+        }
+        //
+        return result;
+    }
+
+    public void setLevel(final String level)
+    {
+        this.level = level;
+    }
+
+    public void setTime(final LocalDateTime time)
+    {
+        this.time = time;
+    }
+
+    /**
+     * To string.
+     *
+     * @return the string
+     */
+    @Override
+    public String toString()
+    {
+        String result;
+
+        StringList buffer = new StringList();
+
+        buffer.append("[time=").append(this.time).append("]");
+        buffer.append("[level=").append(this.level).append("]");
+
+        result = buffer.toString();
+
+        //
+        return result;
+    }
+}
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java
index 0089593..6b81ed4 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java
@@ -23,6 +23,8 @@ import java.io.IOException;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
@@ -107,20 +109,91 @@ public class HttpErrorLogAnalyzer
      */
     public void probeLine(final String line)
     {
+        HttpErrorLog log = parseNginxLog(line);
+
+        // logger.info("==================");
+        if (log == null)
+        {
+            logger.warn("LINE IS NOT MATCHING [{}]", line);
+        }
+        else
+        {
+            // logger.info("LINE IS MATCHING [{}]", log);
+            // logger.info(log.getHttpUserAgent().toString());
+
+            // General HTTP access logs.
+            String year = log.getYear();
+            String yearMonth = log.getYearMonth();
+            String yearWeek = log.getYearWeek();
+            String date = log.getDate();
+
+            // metrics.http.hits
+            this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
+        }
+
+        // TODO metrics.http.errors.php
+    }
+
+    /**
+     * Parses the log.
+     *
+     * @param line
+     *            the line
+     * @param pattern
+     *            the pattern
+     * @return the http error log
+     */
+    public static HttpErrorLog parseLog(final String line, final Pattern pattern)
+    {
+        HttpErrorLog result;
+
+        if (pattern == null)
+        {
+            result = null;
+        }
+        else
+        {
+            Matcher matcher = pattern.matcher(line);
+            if (matcher.matches())
+            {
+                result = new HttpErrorLog();
+                result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)));
+                result.setLevel(matcher.group("level"));
+            }
+            else
+            {
+                result = null;
+            }
+        }
+
         //
-        LocalDateTime date;
+        return result;
+    }
 
-        date = LocalDateTime.now();
+    /**
+     * Parses the nginx log.
+     *
+     * @param line
+     *            the line
+     * @return the http error log
+     */
+    public static HttpErrorLog parseNginxLog(final String line)
+    {
+        HttpErrorLog result;
 
-        String year = date.format(DateTimeFormatter.ofPattern("yyyy", Locale.FRANCE));
-        String month = date.format(DateTimeFormatter.ofPattern("yyyy-MM", Locale.FRANCE));
-        String week = date.format(DateTimeFormatter.ofPattern("yyyyWW", Locale.FRANCE));
-        String day = date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
+        // log_format combined '$remote_addr - $remote_user [$time_local] '
+        // '"$request" $status $body_bytes_sent '
+        // '"$http_referer" "$http_user_agent"';
+        // String pattern = "^(?<time>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})
+        // \\[(?<level>[^\\]])\\] .*$";
+        String patternString = "^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$";
 
-        // metrics.http.errors
-        this.counters.inc("metrics.http.errors", year, month, week, day);
+        Pattern pattern = Pattern.compile(patternString);
 
-        // metrics.http.errors.php
+        result = parseLog(line, pattern);
+
+        //
+        return result;
     }
 
     /**
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
index 4a919a2..8448ab3 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
@@ -92,9 +92,9 @@ public class HttpLogIterator
      * @return the http log
      * @throws IOException
      */
-    public HttpLog next() throws IOException
+    public HttpAccessLog next() throws IOException
     {
-        HttpLog result;
+        HttpAccessLog result;
 
         try
         {
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/VisitCounters.java b/src/fr/devinsy/statoolinfos/metrics/http/VisitCounters.java
index a446753..451d622 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/VisitCounters.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/VisitCounters.java
@@ -59,7 +59,7 @@ public class VisitCounters extends HashMap<String, Visits>
      *            the time mark
      * @return the string
      */
-    public String computeKey(final HttpLog log)
+    public String computeKey(final HttpAccessLog log)
     {
         String result;
 
@@ -136,7 +136,7 @@ public class VisitCounters extends HashMap<String, Visits>
      * @param log
      *            the log
      */
-    public void putVisit(final HttpLog log)
+    public void putVisit(final HttpAccessLog log)
     {
         if (log != null)
         {
diff --git a/test/fr/devinsy/statoolinfos/metrics/http/VisitCountersTest.java b/test/fr/devinsy/statoolinfos/metrics/http/VisitCountersTest.java
index bae70a0..9c78ebf 100644
--- a/test/fr/devinsy/statoolinfos/metrics/http/VisitCountersTest.java
+++ b/test/fr/devinsy/statoolinfos/metrics/http/VisitCountersTest.java
@@ -51,7 +51,7 @@ public class VisitCountersTest
         VisitCounters visitCounters = new VisitCounters();
 
         //
-        HttpLog log = new HttpLog();
+        HttpAccessLog log = new HttpAccessLog();
         log.setRemoteAddress("192.168.1.1");
         log.setTime(LocalDateTime.of(2020, 10, 20, 23, 0));
         log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
@@ -83,14 +83,14 @@ public class VisitCountersTest
 
         //
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0));
             visitCounters.putVisit(log);
         }
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20));
@@ -123,21 +123,21 @@ public class VisitCountersTest
 
         //
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0));
             visitCounters.putVisit(log);
         }
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20));
             visitCounters.putVisit(log);
         }
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 55));
@@ -170,21 +170,21 @@ public class VisitCountersTest
 
         //
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0));
             visitCounters.putVisit(log);
         }
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 55));
             visitCounters.putVisit(log);
         }
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20));
@@ -217,21 +217,21 @@ public class VisitCountersTest
 
         //
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0));
             visitCounters.putVisit(log);
         }
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 35));
             visitCounters.putVisit(log);
         }
         {
-            HttpLog log = new HttpLog();
+            HttpAccessLog log = new HttpAccessLog();
             log.setRemoteAddress("192.168.1.1");
             log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36"));
             log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20));