From 02281b1ab60dce990133fd017c24c382b16e5ce1 Mon Sep 17 00:00:00 2001
From: "Christian P. MOMON" <christian.momon@devinsy.fr>
Date: Mon, 3 Jul 2023 17:40:10 +0200
Subject: [PATCH] Added HttpAccessLogDateTimePattern management (#sleto).

---
 README.md                                     |  4 ++
 .../statoolinfos/core/Configuration.java      | 15 +++++++
 .../statoolinfos/core/StatoolInfos.java       |  4 +-
 .../devinsy/statoolinfos/metrics/Prober.java  | 20 ++++++---
 .../metrics/http/HttpAccessLogIterator.java   | 40 ++++++++++++++----
 .../metrics/http/HttpAccessLogParser.java     | 42 +++++++++++++++++--
 .../metrics/http/HttpAccessLogs.java          | 16 +++----
 .../metrics/http/HttpLogIterator.java         |  4 +-
 8 files changed, 120 insertions(+), 25 deletions(-)

diff --git a/README.md b/README.md
index 809a788..45a2be7 100644
--- a/README.md
+++ b/README.md
@@ -281,6 +281,10 @@ conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.
 # Default: ^(?<remoteAddress>[a-fA-F0-9\:\.]+) - (?<remoteUser>[^\[]+) \[(?<time>[^\]]+)\] "(?<request>.*)" (?<status>\d+) (?<bodyBytesSent>\d+) "(?<referer>.*)" "(?<userAgent>[^"]*)".*$ 
 conf.probe.httpaccesslog.pattern=
 
+# Custom datetime access log pattern (https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html#patterns).
+# Default: dd/MMM/yyyy:HH:mm:ss Z|EN
+conf.prob.httpaccesslog.datetimepattern=
+
 # Filter request with Java regex (without the initial '^' and without the final '$', see https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html).
 # Example: pathfilter=/.*well-known.*
 conf.probe.httpaccesslog.pathfilter=
diff --git a/src/fr/devinsy/statoolinfos/core/Configuration.java b/src/fr/devinsy/statoolinfos/core/Configuration.java
index e3ff853..a73d26e 100644
--- a/src/fr/devinsy/statoolinfos/core/Configuration.java
+++ b/src/fr/devinsy/statoolinfos/core/Configuration.java
@@ -354,6 +354,21 @@ public class Configuration extends PathPropertyList
         return result;
     }
 
+    /**
+     * Gets the probe http access log date pattern.
+     *
+     * @return the probe http access log date pattern
+     */
+    public String getProbeHttpAccessLogDateTimePattern()
+    {
+        String result;
+
+        result = get("conf.probe.httpaccesslog.datetimepattern");
+
+        //
+        return result;
+    }
+
     /**
      * Gets the probe http access log path filter.
      *
diff --git a/src/fr/devinsy/statoolinfos/core/StatoolInfos.java b/src/fr/devinsy/statoolinfos/core/StatoolInfos.java
index 542fe78..a1e1c41 100644
--- a/src/fr/devinsy/statoolinfos/core/StatoolInfos.java
+++ b/src/fr/devinsy/statoolinfos/core/StatoolInfos.java
@@ -295,13 +295,15 @@ public class StatoolInfos
 
                 logger.info("== Testing HttpAccessLog lines.");
                 String source = configuration.getProbeHttpAccessLogSource();
+                String dateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
                 String pattern = configuration.getProbeHttpAccessLogPattern();
                 String pathFilter = configuration.getProbeHttpAccessLogPathFilter();
                 logger.info("source=[{}]", source);
                 logger.info("pattern=[{}]", pattern);
+                logger.info("dateTimePattern=[{}]", dateTimePattern);
                 logger.info("pathFilter=[{}]", pathFilter);
 
-                HttpAccessLogs logs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, pathFilter);
+                HttpAccessLogs logs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, dateTimePattern, pathFilter);
 
                 for (HttpAccessLog log : logs)
                 {
diff --git a/src/fr/devinsy/statoolinfos/metrics/Prober.java b/src/fr/devinsy/statoolinfos/metrics/Prober.java
index 1ba9c9e..844b26d 100644
--- a/src/fr/devinsy/statoolinfos/metrics/Prober.java
+++ b/src/fr/devinsy/statoolinfos/metrics/Prober.java
@@ -227,11 +227,13 @@ public class Prober
 
         String httpLogs = configuration.getProbeHttpAccessLogSource();
         String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
+        String httpAccessLogDateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
         String httpAccessPathFilter = configuration.getProbeHttpAccessLogPathFilter();
         logger.info("httpLogs=[{}]", httpLogs);
         logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
+        logger.info("httpAccessDateTimePattern=[{}]", httpAccessLogDateTimePattern);
         logger.info("httpAccessPathFilter=[{}]", httpAccessPathFilter);
-        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessPathFilter);
+        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessLogDateTimePattern, httpAccessPathFilter);
 
         result = EtherpadProber.probe(httpAccessLogs, FilesUtils.searchByWildcard(logs), database);
 
@@ -296,11 +298,13 @@ public class Prober
 
         String httpLogs = configuration.getProbeHttpAccessLogSource();
         String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
+        String httpAccessLogDateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
         String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
         logger.info("httpLogs=[{}]", httpLogs);
         logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
+        logger.info("httpAccessDateTimePattern=[{}]", httpAccessLogDateTimePattern);
         logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
-        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessLogPathFilter);
+        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpLogs), httpAccessLogPattern, httpAccessLogDateTimePattern, httpAccessLogPathFilter);
 
         result = GiteaProber.probe(httpAccessLogs, apiURL, apiToken, dataDirectory, database);
 
@@ -352,12 +356,14 @@ public class Prober
         logger.info("== Probing HttpAccessLog.");
         String source = configuration.getProbeHttpAccessLogSource();
         String pattern = configuration.getProbeHttpAccessLogPattern();
+        String dateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
         String pathFilter = configuration.getProbeHttpAccessLogPathFilter();
         logger.info("source=[{}]", source);
         logger.info("pattern=[{}]", pattern);
+        logger.info("dateTimePattern=[{}]", dateTimePattern);
         logger.info("path=[{}]", pathFilter);
 
-        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, pathFilter);
+        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(source), pattern, dateTimePattern, pathFilter);
 
         result = HttpAccessLogAnalyzer.probe(httpAccessLogs);
 
@@ -398,11 +404,13 @@ public class Prober
 
         String httpAccessSource = configuration.getProbeHttpAccessLogSource();
         String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
+        String httpAccessLogDateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
         String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
         logger.info("httpAccessSource=[{}]", httpAccessSource);
         logger.info("httpAccessPattern=[{}]", httpAccessLogPattern);
+        logger.info("httpAccessDateTimePattern=[{}]", httpAccessLogDateTimePattern);
         logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
-        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessSource), httpAccessLogPattern, httpAccessLogPathFilter);
+        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessSource), httpAccessLogPattern, httpAccessLogDateTimePattern, httpAccessLogPathFilter);
 
         File dataDirectory = configuration.getAsFile("conf.probe.libreqr.datafiles");
         logger.info("dataDirectory=[{}]", dataDirectory);
@@ -485,11 +493,13 @@ public class Prober
         logger.info("== Probing Privatebin.");
         String httpAccessLogSource = configuration.getProbeHttpAccessLogSource();
         String httpAccessLogPattern = configuration.getProbeHttpAccessLogPattern();
+        String httpAccessLogDateTimePattern = configuration.getProbeHttpAccessLogDateTimePattern();
         String httpAccessLogPathFilter = configuration.getProbeHttpAccessLogPathFilter();
         logger.info("source=[{}]", httpAccessLogSource);
         logger.info("pattern=[{}]", httpAccessLogPattern);
+        logger.info("httpAccessDateTimePattern=[{}]", httpAccessLogDateTimePattern);
         logger.info("httpAccessPath=[{}]", httpAccessLogPathFilter);
-        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessLogSource), httpAccessLogPattern, httpAccessLogPathFilter);
+        HttpAccessLogs httpAccessLogs = new HttpAccessLogs(FilesUtils.searchByWildcard(httpAccessLogSource), httpAccessLogPattern, httpAccessLogDateTimePattern, httpAccessLogPathFilter);
 
         File dataDirectory = configuration.getAsFile("conf.probe.privatebin.data");
         logger.info("dataDirectory=[{}]", dataDirectory);
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java
index bb741e3..04f34bd 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogIterator.java
@@ -19,7 +19,9 @@
 package fr.devinsy.statoolinfos.metrics.http;
 
 import java.io.IOException;
+import java.time.format.DateTimeFormatter;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
@@ -38,6 +40,7 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
 
     private FilesLineIterator lineIterator;
     private Pattern pattern;
+    private DateTimeFormatter dateTimeFormatter;
     private Pattern requestFilter;
     private HttpAccessLog nextLog;
     private int errorCount;
@@ -53,7 +56,7 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
      */
     public HttpAccessLogIterator(final Files source) throws IOException
     {
-        this(source, null, null);
+        this(source, null, null, null);
     }
 
     /**
@@ -68,31 +71,54 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
      */
     public HttpAccessLogIterator(final Files source, final String regex) throws IOException
     {
-        this(source, regex, null);
+        this(source, regex, null, null);
     }
 
     /**
-     * Instantiates a new http log iterator.
+     * Instantiates a new http access log iterator.
      *
      * @param source
      *            the source
+     * @param linePattern
+     *            the line pattern
+     * @param dateTimePattern
+     *            the date time pattern
+     * @param pathFilter
+     *            the path filter
      * @throws IOException
      *             Signals that an I/O exception has occurred.
      */
-    public HttpAccessLogIterator(final Files source, final String regex, final String pathFilter) throws IOException
+    public HttpAccessLogIterator(final Files source, final String linePattern, final String dateTimePattern, final String pathFilter) throws IOException
     {
         this.lineIterator = new FilesLineIterator(source);
         this.nextLog = null;
         this.errorCount = 0;
         this.ignoredLineCount = 0;
 
-        if (StringUtils.isBlank(regex))
+        if (StringUtils.isBlank(linePattern))
         {
             this.pattern = HttpAccessLogParser.COMBINED_PATTERN;
         }
         else
         {
-            this.pattern = Pattern.compile(regex);
+            this.pattern = Pattern.compile(linePattern);
+        }
+
+        if (StringUtils.isBlank(dateTimePattern))
+        {
+            this.dateTimeFormatter = HttpAccessLogParser.DATETIME_FORMATTER;
+        }
+        else
+        {
+            String[] split = dateTimePattern.split("\\|");
+            if (split.length != 2)
+            {
+                throw new IllegalArgumentException("Bad dateTimePattern format: [" + dateTimePattern + "].");
+            }
+            else
+            {
+                this.dateTimeFormatter = DateTimeFormatter.ofPattern(split[0]).withLocale(Locale.forLanguageTag(split[1]));
+            }
         }
 
         if (StringUtils.isBlank(pathFilter))
@@ -183,7 +209,7 @@ public class HttpAccessLogIterator implements Iterator<HttpAccessLog>
 
                     try
                     {
-                        HttpAccessLog log = HttpAccessLogParser.parseLog(line, this.pattern);
+                        HttpAccessLog log = HttpAccessLogParser.parseLog(line, this.pattern, this.dateTimeFormatter);
                         if (log == null)
                         {
                             logger.warn("LINE IS NOT MATCHING [{}]", line);
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogParser.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogParser.java
index e0f9860..249fc5e 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogParser.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogParser.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2020-2022 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -21,6 +21,7 @@ package fr.devinsy.statoolinfos.metrics.http;
 import java.time.LocalDateTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -41,6 +42,8 @@ public class HttpAccessLogParser
     public static final Pattern COMBINED_PATTERN = Pattern.compile(
             "^(?<remoteAddress>[a-fA-F0-9\\:\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>.*)\".*$");
 
+    public static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH);
+
     /**
      * Instantiates a new http access log parser.
      */
@@ -48,6 +51,39 @@ public class HttpAccessLogParser
     {
     }
 
+    /**
+     * Parses the date time.
+     *
+     * @param dateTime
+     *            the date time
+     * @param formatter
+     *            the formatter
+     * @return the zoned date time
+     */
+    public static ZonedDateTime parseDateTime(final CharSequence dateTime, final DateTimeFormatter formatter)
+    {
+        ZonedDateTime result;
+
+        if ((formatter == null) || (formatter == DATETIME_FORMATTER))
+        {
+            result = ZonedDateTime.parse(dateTime, DATETIME_FORMATTER);
+        }
+        else
+        {
+            try
+            {
+                result = ZonedDateTime.parse(dateTime, formatter);
+            }
+            catch (DateTimeParseException exception)
+            {
+                result = ZonedDateTime.parse(dateTime, DATETIME_FORMATTER);
+            }
+        }
+
+        //
+        return result;
+    }
+
     /**
      * Parses the log.
      *
@@ -55,7 +91,7 @@ public class HttpAccessLogParser
      *            the line
      * @return the http log
      */
-    public static HttpAccessLog parseLog(final String line, final Pattern pattern)
+    public static HttpAccessLog parseLog(final String line, final Pattern pattern, final DateTimeFormatter dateTimeFormatter)
     {
         HttpAccessLog result;
 
@@ -68,7 +104,7 @@ public class HttpAccessLogParser
             result = new HttpAccessLog();
             result.setIp(matcher.group("remoteAddress"));
             result.setRemoteUser(matcher.group("remoteUser"));
-            result.setTime(ZonedDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH)));
+            result.setTime(parseDateTime(matcher.group("time"), dateTimeFormatter));
             result.setRequest(matcher.group("request"));
             result.setStatus(HttpStatusTable.instance().get(matcher.group("status")));
             result.setBodyBytesSent(Long.valueOf(matcher.group("bodyBytesSent")));
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java
index 7788cec..720d4b1 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogs.java
@@ -36,6 +36,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
 
     private Files source;
     private String pattern;
+    private String datePattern;
     private String pathFilter;
 
     /**
@@ -48,7 +49,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
      */
     public HttpAccessLogs(final File source) throws IOException
     {
-        this(new Files(source), null, null);
+        this(new Files(source), null, null, null);
     }
 
     /**
@@ -63,7 +64,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
      */
     public HttpAccessLogs(final File source, final String pattern) throws IOException
     {
-        this(new Files(source), pattern, null);
+        this(new Files(source), pattern, null, null);
     }
 
     /**
@@ -80,7 +81,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
      */
     public HttpAccessLogs(final File source, final String pattern, final String pathFilter) throws IOException
     {
-        this(new Files(source), pattern, null);
+        this(new Files(source), pattern, null, null);
     }
 
     /**
@@ -93,7 +94,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
      */
     public HttpAccessLogs(final Files source) throws IOException
     {
-        this(source, null, null);
+        this(source, null, null, null);
     }
 
     /**
@@ -106,7 +107,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
      */
     public HttpAccessLogs(final Files source, final String pattern) throws IOException
     {
-        this(source, pattern, null);
+        this(source, pattern, null, null);
     }
 
     /**
@@ -121,11 +122,12 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
      * @throws IOException
      *             Signals that an I/O exception has occurred.
      */
-    public HttpAccessLogs(final Files source, final String pattern, final String pathFilter) throws IOException
+    public HttpAccessLogs(final Files source, final String pattern, final String datePattern, final String pathFilter) throws IOException
     {
         this.source = source;
         this.pattern = pattern;
         this.pathFilter = pathFilter;
+        this.datePattern = datePattern;
     }
 
     /* (non-Javadoc)
@@ -138,7 +140,7 @@ public class HttpAccessLogs implements Iterable<HttpAccessLog>
 
         try
         {
-            result = new HttpAccessLogIterator(this.source, this.pattern, this.pathFilter);
+            result = new HttpAccessLogIterator(this.source, this.pattern, this.datePattern, this.pathFilter);
         }
         catch (IOException exception)
         {
diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
index 01f535b..55b6b0d 100644
--- a/src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
+++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2021-2022 Christian Pierre MOMON <christian@momon.org>
+ * Copyright (C) 2021-2023 Christian Pierre MOMON <christian@momon.org>
  * 
  * This file is part of StatoolInfos, simple service statistics tool.
  * 
@@ -97,7 +97,7 @@ public class HttpLogIterator
         try
         {
             String line = this.iterator.next();
-            result = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN);
+            result = HttpAccessLogParser.parseLog(line, HttpAccessLogParser.COMBINED_PATTERN, HttpAccessLogParser.DATETIME_FORMATTER);
         }
         catch (Exception exception)
         {