Improved help message, custom pattern log support and Apache patterns.

src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java

@@ -25,7 +25,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import fr.devinsy.statoolinfos.core.LogFilter;
+import fr.devinsy.statoolinfos.core.BotFilter;
 import fr.devinsy.statoolinfos.core.StatoolInfos;
 import fr.devinsy.statoolinfos.util.BuildInformation;
 import fr.devinsy.statoolinfos.util.Chrono;
@@ -104,20 +104,22 @@ public final class StatoolInfosCLI
         message.appendln("Usage:");
         message.appendln("    statoolinfos [ -h | -help | --help ]");
         message.appendln("    statoolinfos [ -v | -version | --version ]");
-        message.appendln("    statoolinfos build                       <fileordirectory>       build property files from conf and input");
+        message.appendln();
-        message.appendln("    statoolinfos clear                       <fileordirectory>       remove property files from conf");
+        message.appendln("    statoolinfos build                       <configurationfile>     build property files from conf and input");
-        message.appendln("    statoolinfos crawl                       <fileordirectory>       crawl all file from conf and input");
+        message.appendln("    statoolinfos clear                       <configurationfile>     remove property files from conf");
-        message.appendln("    statoolinfos format                      <fileordirectory>       format in tiny way");
+        message.appendln("    statoolinfos crawl                       <configurationfile>     crawl all file from conf and input");
-        message.appendln("    statoolinfos htmlize                     <fileordirectory>       generate web pages from conf");
+        message.appendln("    statoolinfos htmlize                     <configurationfile>     generate web pages from conf");
+        message.appendln("    statoolinfos probe                       <configurationfile>     generate metrics files from conf");
+        message.appendln("    statoolinfos uptime                      <configurationfile>     update uptime journal");
+        message.appendln();
+        message.appendln("    statoolinfos format                      <fileordirectory>       format property files in tiny way");
         message.appendln("    statoolinfos list ip       [-bot|-nobot] <fileordirectory>       generate ip list from log file");
         message.appendln("    statoolinfos list ua       [-bot|-nobot] <fileordirectory>       generate user agent list from log file");
         message.appendln("    statoolinfos list visitors [-bot|-nobot] <fileordirectory>       generate visitors (ip+ua) list from log file");
-        message.appendln("    statoolinfos probe                       <fileordirectory>       generate metrics files from conf");
         message.appendln("    statoolinfos stat ip       [-bot|-nobot] <fileordirectory>       generate stats about ip from log file");
         message.appendln("    statoolinfos stat ua       [-bot|-nobot] <fileordirectory>       generate stats about user agent from log file");
         message.appendln("    statoolinfos stat visitors [-bot|-nobot] <fileordirectory>       generate stats about visitors (ip+ua) from log file");
         message.appendln("    statoolinfos tagdate                     <fileordirectory>       update the file.datetime file");
-        message.appendln("    statoolinfos uptime                      <fileordirectory>       update uptime journal");
 
         System.out.println(message.toString());
     }
@@ -200,21 +202,21 @@ public final class StatoolInfosCLI
      *            the source
      * @return the log filter
      */
-    private static LogFilter parseLogFilterOption(final String source)
+    private static BotFilter parseLogFilterOption(final String source)
     {
-        LogFilter result;
+        BotFilter result;
 
         if (StringUtils.equals(source, "-all"))
         {
-            result = LogFilter.ALL;
+            result = BotFilter.ALL;
         }
         else if (StringUtils.equals(source, "-bot"))
         {
-            result = LogFilter.BOT;
+            result = BotFilter.BOT;
         }
         else if (StringUtils.equals(source, "-nobot"))
         {
-            result = LogFilter.NOBOT;
+            result = BotFilter.NOBOT;
         }
         else
         {
@@ -271,52 +273,44 @@ public final class StatoolInfosCLI
         {
             displayVersion();
         }
-        else if (isMatching(args, "build", "\\s*.+\\s*"))
+        else if (isMatching(args, "build", "\\s*.+\\.conf\\s*"))
-        {
-            Files inputs = convertPath(StringUtils.trim(args[1]));
-            for (File input : inputs)
         {
+            File configurationFile = new File(StringUtils.trim(args[1]));
             try
             {
-                    StatoolInfos.build(input);
+                StatoolInfos.build(configurationFile);
             }
             catch (Exception exception)
             {
-                    logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
+                logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
             }
         }
-        }
+        else if (isMatching(args, "clear", "\\s*.+\\.conf\\s*"))
-        else if (isMatching(args, "clear", "\\s*.+\\s*"))
-        {
-            Files inputs = convertPath(StringUtils.trim(args[1]));
-            for (File input : inputs)
         {
+            File configurationFile = new File(StringUtils.trim(args[1]));
             try
             {
-                    StatoolInfos.clear(input);
+                StatoolInfos.clear(configurationFile);
             }
             catch (Exception exception)
             {
-                    logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
+                logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
             }
         }
-        }
+        else if (isMatching(args, "crawl", "\\s*.+\\.conf\\s*"))
-        else if (isMatching(args, "crawl", "\\s*.+\\s*"))
         {
             Chrono chrono = new Chrono().start();
-            Files inputs = convertPath(StringUtils.trim(args[1]));
+
-            for (File input : inputs)
+            File configurationFile = new File(StringUtils.trim(args[1]));
-            {
             try
             {
-                    StatoolInfos.crawl(input);
+                StatoolInfos.crawl(configurationFile);
             }
             catch (Exception exception)
             {
-                    logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
+                logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
                 exception.printStackTrace();
             }
-            }
             System.out.println(chrono.format());
         }
         else if (isMatching(args, "format", "\\s*.+\\s*"))
@@ -335,33 +329,30 @@ public final class StatoolInfosCLI
                 }
             }
         }
-        else if (isMatching(args, "htmlize", "\\s*.+\\s*"))
+        else if (isMatching(args, "htmlize", "\\s*.+\\.conf\\s*"))
         {
             Chrono chrono = new Chrono().start();
-            Files inputs = convertPath(StringUtils.trim(args[1]));
+            File configurationFile = new File(StringUtils.trim(args[1]));
-            for (File input : inputs)
-            {
             try
             {
-                    StatoolInfos.htmlize(input);
+                StatoolInfos.htmlize(configurationFile);
             }
             catch (Exception exception)
             {
-                    logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
+                logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
                 exception.printStackTrace();
             }
-            }
             System.out.println(chrono.format());
         }
         else if (isMatching(args, "list", "ip", "\\s*\\S+\\s*"))
         {
             File source = new File(args[2]);
 
-            StatoolInfos.listIps(source, LogFilter.ALL);
+            StatoolInfos.listIps(source, BotFilter.ALL);
         }
         else if (isMatching(args, "list", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
         {
-            LogFilter filter = parseLogFilterOption(args[2]);
+            BotFilter filter = parseLogFilterOption(args[2]);
             File source = new File(args[3]);
 
             StatoolInfos.listIps(source, filter);
@@ -370,11 +361,11 @@ public final class StatoolInfosCLI
         {
             File source = new File(args[2]);
 
-            StatoolInfos.listUserAgents(source, LogFilter.ALL);
+            StatoolInfos.listUserAgents(source, BotFilter.ALL);
         }
         else if (isMatching(args, "list", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
         {
-            LogFilter filter = parseLogFilterOption(args[2]);
+            BotFilter filter = parseLogFilterOption(args[2]);
             File source = new File(args[3]);
 
             StatoolInfos.listUserAgents(source, filter);
@@ -383,40 +374,37 @@ public final class StatoolInfosCLI
         {
             File source = new File(args[2]);
 
-            StatoolInfos.listVisitors(source, LogFilter.ALL);
+            StatoolInfos.listVisitors(source, BotFilter.ALL);
         }
         else if (isMatching(args, "list", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
         {
-            LogFilter filter = parseLogFilterOption(args[2]);
+            BotFilter filter = parseLogFilterOption(args[2]);
             File source = new File(args[3]);
 
             StatoolInfos.listVisitors(source, filter);
         }
-        else if (isMatching(args, "probe", "\\s*.+\\s*"))
+        else if (isMatching(args, "probe", "\\s*.+\\.conf\\s*"))
-        {
-            Files inputs = convertPath(StringUtils.trim(args[1]));
-            for (File input : inputs)
         {
+            File configurationFile = new File(StringUtils.trim(args[1]));
             try
             {
-                    StatoolInfos.probe(input);
+                StatoolInfos.probe(configurationFile);
             }
             catch (Exception exception)
             {
-                    logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
+                logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
                 exception.printStackTrace();
             }
         }
-        }
         else if (isMatching(args, "stat", "ip", "\\s*\\S+\\s*"))
         {
             File source = new File(args[2]);
 
-            StatoolInfos.statIps(source, LogFilter.ALL);
+            StatoolInfos.statIps(source, BotFilter.ALL);
         }
         else if (isMatching(args, "stat", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
         {
-            LogFilter filter = parseLogFilterOption(args[2]);
+            BotFilter filter = parseLogFilterOption(args[2]);
             File source = new File(args[3]);
 
             StatoolInfos.statIps(source, filter);
@@ -424,11 +412,11 @@ public final class StatoolInfosCLI
         else if (isMatching(args, "stat", "(useragent|ua)", "\\s*\\S+\\s*"))
         {
             File source = new File(args[2]);
-            StatoolInfos.statUserAgents(source, LogFilter.ALL);
+            StatoolInfos.statUserAgents(source, BotFilter.ALL);
         }
         else if (isMatching(args, "stat", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
         {
-            LogFilter filter = parseLogFilterOption(args[2]);
+            BotFilter filter = parseLogFilterOption(args[2]);
             File source = new File(args[3]);
 
             StatoolInfos.statUserAgents(source, filter);
@@ -437,11 +425,11 @@ public final class StatoolInfosCLI
         {
             File source = new File(args[2]);
 
-            StatoolInfos.statVisitors(source, LogFilter.ALL);
+            StatoolInfos.statVisitors(source, BotFilter.ALL);
         }
         else if (isMatching(args, "stat", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*"))
         {
-            LogFilter filter = parseLogFilterOption(args[2]);
+            BotFilter filter = parseLogFilterOption(args[2]);
             File source = new File(args[3]);
 
             StatoolInfos.statVisitors(source, filter);
@@ -462,22 +450,20 @@ public final class StatoolInfosCLI
                 }
             }
         }
-        else if (isMatching(args, "uptime", "\\s*.+\\s*"))
+        else if (isMatching(args, "uptime", "\\s*.+\\.conf\\s*"))
         {
             Chrono chrono = new Chrono().start();
-            Files inputs = convertPath(StringUtils.trim(args[1]));
+
-            for (File input : inputs)
+            File configurationFile = new File(StringUtils.trim(args[1]));
-            {
             try
             {
-                    StatoolInfos.uptime(input);
+                StatoolInfos.uptime(configurationFile);
             }
             catch (Exception exception)
             {
-                    logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
+                logger.error("Error with [{}]: {}", configurationFile.getAbsoluteFile(), exception.getMessage());
                 exception.printStackTrace();
             }
-            }
             System.out.println(chrono.format());
         }
         else

src/fr/devinsy/statoolinfos/core/BotFilter.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple service statistics tool.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.core;
+
+import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog;
+
+/**
+ * The Enum LogFilter.
+ */
+public enum BotFilter
+{
+    ALL,
+    BOT,
+    NOBOT;
+
+    /**
+     * Matches.
+     *
+     * @param log
+     *            the log
+     * @return true, if successful
+     */
+    public boolean matches(final HttpAccessLog log)
+    {
+        boolean result;
+
+        if (log == null)
+        {
+            result = false;
+        }
+        else if (this == ALL)
+        {
+            result = true;
+        }
+        else if ((this == BOT) && (log.isBot()))
+        {
+            result = true;
+        }
+        else if ((this == NOBOT) && (!log.isBot()))
+        {
+            result = true;
+        }
+        else
+        {
+            result = false;
+        }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/statoolinfos/core/Configuration.java

@@ -331,6 +331,21 @@ public class Configuration extends PathPropertyList
         return result;
     }
 
+    /**
+     * Gets the probe http access log pattern.
+     *
+     * @return the probe http access log pattern
+     */
+    public String getProbeHttpAccessLogPattern()
+    {
+        String result;
+
+        result = get("conf.probe.httpaccesslog.pattern");
+
+        //
+        return result;
+    }
+
     /**
      * Gets the probe http access log file.
      *

src/fr/devinsy/statoolinfos/core/StatoolInfos.java

@@ -169,7 +169,7 @@ public class StatoolInfos
      *            the source
      * @throws IOException
      */
-    public static void listIps(final File source, final LogFilter filter)
+    public static void listIps(final File source, final BotFilter filter)
     {
         try
         {
@@ -180,7 +180,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
+                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
 
                 if (filter.matches(log))
                 {
@@ -219,7 +219,7 @@ public class StatoolInfos
      * @param source
      *            the source
      */
-    public static void listUserAgents(final File source, final LogFilter filter)
+    public static void listUserAgents(final File source, final BotFilter filter)
     {
         try
         {
@@ -230,7 +230,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
+                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
 
                 if (filter.matches(log))
                 {
@@ -270,7 +270,7 @@ public class StatoolInfos
      * @param source
      *            the source
      */
-    public static void listVisitors(final File source, final LogFilter filter)
+    public static void listVisitors(final File source, final BotFilter filter)
     {
         try
         {
@@ -281,7 +281,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
+                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
 
                 if (filter.matches(log))
                 {
@@ -336,7 +336,7 @@ public class StatoolInfos
      * @param source
      *            the source
      */
-    public static void statIps(final File source, final LogFilter filter)
+    public static void statIps(final File source, final BotFilter filter)
     {
         try
         {
@@ -347,7 +347,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
+                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
                 if (filter.matches(log))
                 {
                     stator.putLog(log);
@@ -387,7 +387,7 @@ public class StatoolInfos
      * @param source
      *            the source
      */
-    public static void statUserAgents(final File source, final LogFilter filter)
+    public static void statUserAgents(final File source, final BotFilter filter)
     {
         try
         {
@@ -398,7 +398,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
+                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
                 if (filter.matches(log))
                 {
                     stator.putLog(log);
@@ -438,7 +438,7 @@ public class StatoolInfos
      * @param source
      *            the source
      */
-    public static void statVisitors(final File source, final LogFilter filter)
+    public static void statVisitors(final File source, final BotFilter filter)
     {
         try
         {
@@ -449,7 +449,7 @@ public class StatoolInfos
             while (iterator.hasNext())
             {
                 String line = iterator.next();
-                HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
+                HttpAccessLog log = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
                 if (filter.matches(log))
                 {
                     stator.putLog(log);

src/fr/devinsy/statoolinfos/metrics/Prober.java

@@ -75,9 +75,11 @@ public class Prober
         {
             logger.info("== Processing HttpAccessLog.");
             String source = configuration.getProbeHttpAccessLogSource();
+            String patternRegex = configuration.getProbeHttpAccessLogPattern();
             logger.info("source=[{}]", source);
+            logger.info("pattern=[{}]", patternRegex);
 
-            PathCounters data = HttpAccessLogAnalyzer.probe(source);
+            PathCounters data = HttpAccessLogAnalyzer.probe(source, patternRegex);
             counters.putAll(data);
         }
 
@@ -176,15 +178,19 @@ public class Prober
 
             for (String year : years)
             {
-                {
+                // Year stat is complicated to build because needs all the
-                    // Year.
+                // log of one year.
-                    PathCounter yearCounter = prefixCounters.get(prefix, year);
+                // {
-                    if (yearCounter != null)
+                // // Year.
-                    {
+                // PathCounter yearCounter = prefixCounters.get(prefix, year);
-                        String line = String.format("%s.%s=%s", yearCounter.getPath(), yearCounter.getTimeMark(), yearCounter.getCounter());
+                // if (yearCounter != null)
-                        metrics.appendln(line);
+                // {
-                    }
+                // String line = String.format("%s.%s=%s",
-                }
+                // yearCounter.getPath(), yearCounter.getTimeMark(),
+                // yearCounter.getCounter());
+                // metrics.appendln(line);
+                // }
+                // }
 
                 {
                     // Months.

src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java

@@ -26,13 +26,13 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import fr.devinsy.statoolinfos.core.StatoolInfosException;
 import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.util.FilesUtils;
 import fr.devinsy.statoolinfos.util.LineIterator;
 
 /**
@@ -44,6 +44,12 @@ public class HttpAccessLogAnalyzer
 
     public static final String DEFAULT_CHARSET_NAME = "UTF-8";
 
+    // log_format combined '$remote_addr - $remote_user [$time_local] '
+    // '"$request" $status $body_bytes_sent '
+    // '"$http_referer" "$http_user_agent"';
+    public static final Pattern COMBINED_PATTERN = Pattern.compile(
+            "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
+
     private int errorCount;
     private PathCounters counters;
     private VisitCounters visits;
@@ -124,9 +130,22 @@ public class HttpAccessLogAnalyzer
      * @throws StatoolInfosException
      *             the statool infos exception
      */
-    public void probe(final File file) throws IOException
+    public void probe(final File file, final String patternRegex) throws IOException
     {
         System.out.println("Probing file [" + file.getAbsolutePath() + "]");
+
+        //
+        Pattern pattern;
+        if (patternRegex == null)
+        {
+            pattern = COMBINED_PATTERN;
+        }
+        else
+        {
+            pattern = Pattern.compile(patternRegex);
+        }
+
+        //
         LineIterator iterator = new LineIterator(file);
         while (iterator.hasNext())
         {
@@ -134,7 +153,15 @@ public class HttpAccessLogAnalyzer
 
             try
             {
-                probeLine(line);
+                HttpAccessLog log = parseLog(line, pattern);
+                if (log == null)
+                {
+                    logger.warn("LINE IS NOT MATCHING [{}]", line);
+                }
+                else
+                {
+                    probeLog(log);
+                }
             }
             catch (Exception exception)
             {
@@ -151,16 +178,10 @@ public class HttpAccessLogAnalyzer
      * @param line
      *            the line
      */
-    public void probeLine(final String line)
+    public void probeLog(final HttpAccessLog log)
     {
-        HttpAccessLog log = parseNginxCombinedLog(line);
-
         // logger.info("==================");
-        if (log == null)
+        if (log != null)
-        {
-            logger.warn("LINE IS NOT MATCHING [{}]", line);
-        }
-        else
         {
             // logger.info("LINE IS MATCHING [{}]", log);
             // logger.info(log.getHttpUserAgent().toString());
@@ -384,30 +405,6 @@ public class HttpAccessLogAnalyzer
         return result;
     }
 
-    /**
-     * Parses the log.
-     *
-     * @param line
-     *            the line
-     * @return the http log
-     */
-    public static HttpAccessLog parseNginxCombinedLog(final String line)
-    {
-        HttpAccessLog result;
-
-        // log_format combined '$remote_addr - $remote_user [$time_local] '
-        // '"$request" $status $body_bytes_sent '
-        // '"$http_referer" "$http_user_agent"';
-        String combinedPattern = "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$";
-
-        Pattern pattern = Pattern.compile(combinedPattern);
-
-        result = parseLog(line, pattern);
-
-        //
-        return result;
-    }
-
     /**
      * Probe.
      *
@@ -418,30 +415,15 @@ public class HttpAccessLogAnalyzer
      * @throws StatoolInfosException
      *             the statool infos exception
      */
-    public static PathCounters probe(final String source) throws IOException, StatoolInfosException
+    public static PathCounters probe(final String source, final String patternRegex) throws IOException, StatoolInfosException
     {
         PathCounters result;
 
         HttpAccessLogAnalyzer analyzer = new HttpAccessLogAnalyzer();
 
-        if (!StringUtils.isBlank(source))
+        for (File file : FilesUtils.searchByWildcard(source))
         {
-            if (source.endsWith("*"))
+            analyzer.probe(file, patternRegex);
-            {
-                String prefix = source.substring(0, source.length() - 1);
-                for (File file : new File(prefix).getParentFile().listFiles())
-                {
-                    if (file.getName().startsWith(FilenameUtils.getName(prefix)))
-                    {
-                        analyzer.probe(file);
-                    }
-                }
-            }
-            else
-            {
-                File file = new File(source);
-                analyzer.probe(file);
-            }
         }
 
         result = analyzer.getCounters();

src/fr/devinsy/statoolinfos/metrics/http/HttpErrorLogAnalyzer.java

@@ -26,13 +26,12 @@ import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import fr.devinsy.statoolinfos.core.StatoolInfosException;
 import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.util.FilesUtils;
 import fr.devinsy.statoolinfos.util.LineIterator;
 
 /**
@@ -44,6 +43,9 @@ public class HttpErrorLogAnalyzer
 
     public static final String DEFAULT_CHARSET_NAME = "UTF-8";
 
+    public static final Pattern NGINX_ERROR_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
+    public static final Pattern APACHE_ERROR_PATTERN = Pattern.compile("TODO^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
+
     private PathCounters counters;
     private int errorCount;
 
@@ -82,6 +84,20 @@ public class HttpErrorLogAnalyzer
      */
     public void probe(final File file) throws IOException
     {
+        if ((file != null) && (!file.isFile()) || (file.exists()))
+        {
+            //
+            Pattern pattern;
+            if (LineIterator.readFirstLine(file).startsWith("["))
+            {
+                pattern = APACHE_ERROR_PATTERN;
+            }
+            else
+            {
+                pattern = NGINX_ERROR_PATTERN;
+            }
+
+            //
             LineIterator iterator = new LineIterator(file);
             while (iterator.hasNext())
             {
@@ -89,7 +105,16 @@ public class HttpErrorLogAnalyzer
 
                 try
                 {
-                probeLine(line);
+                    HttpErrorLog log = parseLog(line, pattern);
+                    // logger.info("==================");
+                    if (log == null)
+                    {
+                        logger.warn("LINE IS NOT MATCHING [{}]", line);
+                    }
+                    else
+                    {
+                        probeLog(log);
+                    }
                 }
                 catch (Exception exception)
                 {
@@ -99,6 +124,7 @@ public class HttpErrorLogAnalyzer
                 }
             }
         }
+    }
 
     /**
      * Probe line.
@@ -106,20 +132,8 @@ public class HttpErrorLogAnalyzer
      * @param line
      *            the line
      */
-    public void probeLine(final String line)
+    public void probeLog(final HttpErrorLog log)
     {
-        HttpErrorLog log = parseNginxLog(line);
-
-        // logger.info("==================");
-        if (log == null)
-        {
-            logger.warn("LINE IS NOT MATCHING [{}]", line);
-        }
-        else
-        {
-            // logger.info("LINE IS MATCHING [{}]", log);
-            // logger.info(log.getHttpUserAgent().toString());
-
         // General HTTP access logs.
         String year = log.getYear();
         String yearMonth = log.getYearMonth();
@@ -128,7 +142,6 @@ public class HttpErrorLogAnalyzer
 
         // metrics.http.hits
         this.counters.inc("metrics.http.errors", year, yearMonth, yearWeek, date);
-        }
 
         // TODO metrics.http.errors.php
     }
@@ -212,25 +225,10 @@ public class HttpErrorLogAnalyzer
 
         HttpErrorLogAnalyzer analyzer = new HttpErrorLogAnalyzer();
 
-        if (!StringUtils.isBlank(source))
+        for (File file : FilesUtils.searchByWildcard(source))
-        {
-            if (source.endsWith("*"))
-            {
-                String prefix = source.substring(0, source.length() - 1);
-                for (File file : new File(prefix).getParentFile().listFiles())
-                {
-                    if (file.getName().startsWith(FilenameUtils.getName(prefix)))
         {
             analyzer.probe(file);
         }
-                }
-            }
-            else
-            {
-                File file = new File(source);
-                analyzer.probe(file);
-            }
-        }
 
         result = analyzer.getCounters();
 

src/fr/devinsy/statoolinfos/metrics/http/HttpLogIterator.java

@@ -99,7 +99,7 @@ public class HttpLogIterator
         try
         {
             String line = this.iterator.next();
-            result = HttpAccessLogAnalyzer.parseNginxCombinedLog(line);
+            result = HttpAccessLogAnalyzer.parseLog(line, HttpAccessLogAnalyzer.COMBINED_PATTERN);
         }
         catch (Exception exception)
         {

src/fr/devinsy/statoolinfos/util/FilesUtils.java

@@ -21,6 +21,7 @@ package fr.devinsy.statoolinfos.util;
 import java.io.File;
 import java.util.regex.Pattern;
 
+import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.StringUtils;
 
 /**
@@ -104,6 +105,44 @@ public class FilesUtils
         return result;
     }
 
+    /**
+     * Search by wildcard.
+     *
+     * @param source
+     *            the source
+     * @return the files
+     */
+    public static Files searchByWildcard(final String source)
+    {
+        Files result;
+
+        result = new Files();
+
+        if (!StringUtils.isBlank(source))
+        {
+            if (source.endsWith("*"))
+            {
+                String prefix = source.substring(0, source.length() - 1);
+                String shortPrefix = FilenameUtils.getName(prefix);
+                for (File file : new File(prefix).getParentFile().listFiles())
+                {
+                    if (file.getName().startsWith(shortPrefix))
+                    {
+                        result.add(file);
+                    }
+                }
+            }
+            else
+            {
+                File file = new File(source);
+                result.add(file);
+            }
+        }
+
+        //
+        return result;
+    }
+
     /**
      * List recursively.
      *
@@ -131,4 +170,32 @@ public class FilesUtils
         //
         return result;
     }
+
+    /**
+     * Search starting with.
+     *
+     * @param source
+     *            the source
+     * @param extensions
+     *            the extensions
+     * @return the files
+     */
+    public static Files searchStartingWith(final File source, final String... extensions)
+    {
+        Files result;
+
+        result = new Files();
+
+        Files full = listRecursively(source);
+        for (File file : full)
+        {
+            if (StringUtils.startsWithAny(file.getName(), extensions))
+            {
+                result.add(file);
+            }
+        }
+
+        //
+        return result;
+    }
 }

src/fr/devinsy/statoolinfos/util/LineIterator.java

@@ -131,4 +131,42 @@ public class LineIterator
             this.ready = true;
         }
     }
+
+    /**
+     * Read first line.
+     *
+     * @param file
+     *            the file
+     * @return the string
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public static String readFirstLine(final File file) throws IOException
+    {
+        String result;
+
+        LineIterator iterator = null;
+        try
+        {
+            iterator = new LineIterator(file);
+            if (iterator.hasNext())
+            {
+                result = iterator.next();
+            }
+            else
+            {
+                result = null;
+            }
+        }
+        finally
+        {
+            if (iterator != null)
+            {
+                iterator.close();
+            }
+        }
+
+        //
+        return result;
+    }
 }