Added tests about http access logs.

src/fr/devinsy/statoolinfos/metrics/httpaccess/HttpAccessLogParser.java

@@ -40,7 +40,7 @@ public class HttpAccessLogParser
     // '"$request" $status $body_bytes_sent '
     // '"$http_referer" "$http_user_agent"';
     public static final Pattern COMBINED_PATTERN = Pattern.compile(
-            "^(?<remoteAddress>[a-fA-F0-9\\:\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>.*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>.*)\".*$");
+            "^(?<remoteAddress>[a-fA-F0-9\\:\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] \"(?<request>([^\"]|\\\")*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>([^\"]|\\\")*)\" \"(?<userAgent>([^\"]|\\\")*)\".*$");
 
     public static final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH);
 
@@ -84,6 +84,23 @@ public class HttpAccessLogParser
         return result;
     }
 
+    /**
+     * Parses the log.
+     *
+     * @param line
+     *            the line
+     * @return the http access log
+     */
+    public static HttpAccessLog parseLog(final String line)
+    {
+        HttpAccessLog result;
+
+        result = parseLog(line, COMBINED_PATTERN, DATETIME_FORMATTER);
+
+        //
+        return result;
+    }
+
     /**
      * Parses the log.
      *

src/fr/devinsy/statoolinfos/metrics/httpaccess/UserAgentBotDetector.java

@@ -42,7 +42,7 @@ public class UserAgentBotDetector
         StringList lines;
         try
         {
-            lines = StringsUtils.load(UserAgentBotDetector.class.getResource("/fr/devinsy/statoolinfos/metrics/http/userAgentBotDetectorData.txt"));
+            lines = StringsUtils.load(UserAgentBotDetector.class.getResource("/fr/devinsy/statoolinfos/metrics/httpaccess/userAgentBotDetectorData.txt"));
         }
         catch (IOException exception)
         {

test/fr/devinsy/statoolinfos/metrics/httpaccess/HttpAccessLogIteratorTest.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2021-2024 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple key value database.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.httpaccess;
+
+import java.io.File;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.core.config.DefaultConfiguration;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.statoolinfos.util.FilesUtils;
+
+/**
+ * The Class HttpAccessLogIteratorTest.
+ */
+public class HttpAccessLogIteratorTest
+{
+    private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpAccessLogIteratorTest.class);
+
+    /**
+     * Test 01.
+     *
+     * @throws Exception
+     *             the exception
+     */
+    @Test
+    public void test01() throws Exception
+    {
+        System.out.println(System.getProperty("user.dir"));
+
+        String source = "./test/fr/devinsy/statoolinfos/metrics/httpaccess/data/lseu/www*";
+        Files files = FilesUtils.searchByWildcard(source);
+        for (File file : files)
+        {
+            System.out.println(file);
+        }
+        HttpAccessLogs logs = new HttpAccessLogs(files);
+        HttpAccessLogIterator iterator = (HttpAccessLogIterator) logs.iterator();
+        while (iterator.hasNext())
+        {
+            HttpAccessLog log = iterator.next();
+            // System.out.println(log.toStringLog());
+        }
+
+        System.out.println(iterator.getLogCount());
+        System.out.println(iterator.getFailedLogCount());
+
+        Assert.assertEquals(7146, iterator.getLogCount());
+        Assert.assertEquals(0, iterator.getFailedLogCount());
+    }
+
+    /**
+     * After class.
+     * 
+     * @throws StatoolInfosException
+     *             the Juga exception
+     */
+    @AfterClass
+    public static void afterClass() throws StatoolInfosException
+    {
+    }
+
+    /**
+     * Before class.
+     * 
+     * @throws StatoolInfosException
+     *             the Juga exception
+     */
+    @BeforeClass
+    public static void beforeClass() throws StatoolInfosException
+    {
+        Configurator.initialize(new DefaultConfiguration());
+        Configurator.setRootLevel(Level.DEBUG);
+    }
+}

test/fr/devinsy/statoolinfos/metrics/httpaccess/HttpAccessLogParserTest.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2021-2024 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple key value database.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.httpaccess;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.core.config.DefaultConfiguration;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+
+/**
+ * The Class HttpAccessLogParserTest.
+ */
+public class HttpAccessLogParserTest
+{
+    private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpAccessLogParserTest.class);
+
+    /**
+     * Test 01.
+     *
+     * @throws Exception
+     *             the exception
+     */
+    @Test
+    public void test01() throws Exception
+    {
+        // Nominal line.
+        String line = "172.104.137.47 - - [30/Apr/2023:23:25:15 +0200] \"GET /default.jsp HTTP/1.1\" 302 534 \"-\" \"curl/7.54.0\"";
+        Assert.assertNotNull(HttpAccessLogParser.parseLog(line));
+
+        // Line with a \" in request string.
+        line = "172.104.137.47 - - [30/Apr/2023:23:25:15 +0200] \"GET /def\\\"ault.jsp HTTP/1.1\" 302 534 \"-\" \"curl/7.54.0\"";
+        Assert.assertNotNull(HttpAccessLogParser.parseLog(line));
+
+        // Line with many \".
+        line = "10.141.57.46 - - [28/Apr/2023:13:46:34 +0200] \"GET /?\\\"<?=print(9347655345-4954366)?>\\\" HTTP/1.1\" 302 5143 \"https://www.gaagle.com/\\\"<?=print(9347655345-4954366);?>\\\"\" \"Mozilliqa\\\"<?=print(9347655345-4954366);?>\\\"\"";
+        Assert.assertNotNull(HttpAccessLogParser.parseLog(line));
+    }
+
+    /**
+     * After class.
+     * 
+     * @throws StatoolInfosException
+     *             the Juga exception
+     */
+    @AfterClass
+    public static void afterClass() throws StatoolInfosException
+    {
+    }
+
+    /**
+     * Before class.
+     * 
+     * @throws StatoolInfosException
+     *             the Juga exception
+     */
+    @BeforeClass
+    public static void beforeClass() throws StatoolInfosException
+    {
+        Configurator.initialize(new DefaultConfiguration());
+        Configurator.setRootLevel(Level.DEBUG);
+    }
+}

test/fr/devinsy/statoolinfos/metrics/httpaccess/HttpAccessLogsAnalyzerTest.java

@@ -0,0 +1,131 @@
+/*
+ * Copyright (C) 2021-2024 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple key value database.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.httpaccess;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.core.config.DefaultConfiguration;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import fr.devinsy.statoolinfos.core.Configuration;
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+import fr.devinsy.statoolinfos.metrics.PathCounters;
+import fr.devinsy.statoolinfos.metrics.Prober;
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.statoolinfos.util.FilesUtils;
+
+/**
+ * The Class HttpAccessLogsAnalyzerTest.
+ */
+public class HttpAccessLogsAnalyzerTest
+{
+    private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpAccessLogsAnalyzerTest.class);
+
+    /**
+     * @throws Exception
+     */
+    @Test
+    public void test01() throws Exception
+    {
+        // System.out.println(System.getProperty("user.dir"));
+
+        Files files = FilesUtils.searchByWildcard("./test/fr/devinsy/statoolinfos/metrics/httpaccess/data/lseu/www*");
+        // for (File file : files)
+        // {
+        // System.out.println(file);
+        // }
+        HttpAccessLogs logs = new HttpAccessLogs(files);
+
+        PathCounters counters = HttpAccessLogAnalyzer.probe(logs);
+
+        // for (String prefix : counters.getPrefixes())
+        // {
+        // System.out.println(prefix);
+        // }
+
+        System.out.println("Prefix count: " + counters.getPrefixes().size());
+
+        Assert.assertEquals(41, counters.getPrefixes().size());
+        Assert.assertEquals(1035, counters.getCount("metrics.http.hits", "2022-11"));
+        Assert.assertEquals(1031, counters.getCount("metrics.http.hits", "2022-W44"));
+        Assert.assertEquals(964, counters.getCount("metrics.http.hits", "2022-11-01"));
+        Assert.assertEquals(1016, counters.getCount("metrics.http.hits", "2023-01"));
+    }
+
+    /**
+     * @throws Exception
+     */
+    @Test
+    public void test02() throws Exception
+    {
+        // System.out.println(System.getProperty("user.dir"));
+
+        Configuration configuration = new Configuration();
+        configuration.add("conf.probe.types", "HttpAccessLog");
+        configuration.add("conf.probe.httpaccesslog.file", "./test/fr/devinsy/statoolinfos/metrics/httpaccess/data/lseu/www*");
+        // configuration.add("conf.probe.httperrorlog.pattern", "");
+        // configuration.add("conf.probe.httperrorlog.dateTimePattern", "");
+
+        PathCounters counters = Prober.probeHttpAccessLog(configuration);
+
+        // for (String counter : counters.keySet())
+        // {
+        // System.out.println(counter + " " +
+        // counters.get(counter).getCounter());
+        // }
+
+        Assert.assertEquals(983, counters.size());
+
+        //
+        Assert.assertEquals(2056, counters.getCount("metrics.http.hits", "2022"));
+        Assert.assertEquals(1035, counters.getCount("metrics.http.hits", "2022-11"));
+        Assert.assertEquals(1031, counters.getCount("metrics.http.hits", "2022-W44"));
+        Assert.assertEquals(964, counters.getCount("metrics.http.hits", "2022-11-01"));
+
+        Assert.assertEquals(",,,,,,,,,,1035,1021", counters.getMonthsValuesLine("metrics.http.hits", "2022"));
+        Assert.assertEquals(",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1031,,4,,1021,,,,1014", counters.getWeeksValuesLine("metrics.http.hits", "2022"));
+    }
+
+    /**
+     * After class.
+     * 
+     * @throws StatoolInfosException
+     *             the Juga exception
+     */
+    @AfterClass
+    public static void afterClass() throws StatoolInfosException
+    {
+    }
+
+    /**
+     * Before class.
+     * 
+     * @throws StatoolInfosException
+     *             the Juga exception
+     */
+    @BeforeClass
+    public static void beforeClass() throws StatoolInfosException
+    {
+        Configurator.initialize(new DefaultConfiguration());
+        Configurator.setRootLevel(Level.DEBUG);
+    }
+}

test/fr/devinsy/statoolinfos/metrics/httpaccess/data/lseu/README

@@ -0,0 +1,3 @@
+From Apache server.
+
+Anonymize with Logar.

test/fr/devinsy/statoolinfos/metrics/httpaccess/data/notempty

@@ -0,0 +1 @@
+10.222.42.55 - - [01/Dec/2021:01:01:03 +0100] "GET /.well-known/statoolinfos/libre-service.eu.properties HTTP/1.1" 200 6387 "-" "Java/11.0.13"

test/fr/devinsy/statoolinfos/metrics/httperror/HttpErrorLogIteratorTest.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2021-2024 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of StatoolInfos, simple key value database.
+ * 
+ * StatoolInfos is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * StatoolInfos is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with StatoolInfos.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.statoolinfos.metrics.httperror;
+
+import java.io.File;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.config.Configurator;
+import org.apache.logging.log4j.core.config.DefaultConfiguration;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import fr.devinsy.statoolinfos.core.StatoolInfosException;
+import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLog;
+import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogIterator;
+import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogs;
+import fr.devinsy.statoolinfos.util.Files;
+import fr.devinsy.statoolinfos.util.FilesUtils;
+
+/**
+ * The Class HttpErrorLogsTest.
+ */
+public class HttpErrorLogIteratorTest
+{
+    private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpErrorLogIteratorTest.class);
+
+    /**
+     * Test 01.
+     *
+     * @throws Exception
+     *             the exception
+     */
+    @Test
+    public void test01() throws Exception
+    {
+        System.out.println(System.getProperty("user.dir"));
+
+        String source = "./test/fr/devinsy/statoolinfos/metrics/httperror/data/paste.libre-service.eu/paste*";
+        Files files = FilesUtils.searchByWildcard(source);
+        for (File file : files)
+        {
+            System.out.println(file);
+        }
+        HttpErrorLogs logs = new HttpErrorLogs(files);
+        HttpErrorLogIterator iterator = (HttpErrorLogIterator) logs.iterator();
+        while (iterator.hasNext())
+        {
+            HttpErrorLog log = iterator.next();
+            System.out.println(log.toStringLog());
+        }
+
+        // System.out.println(iterator.getLogCount());
+        // System.out.println(iterator.getFailedLogCount());
+
+        Assert.assertEquals(315, iterator.getLogCount());
+        Assert.assertEquals(0, iterator.getFailedLogCount());
+    }
+
+    /**
+     * After class.
+     * 
+     * @throws StatoolInfosException
+     *             the Juga exception
+     */
+    @AfterClass
+    public static void afterClass() throws StatoolInfosException
+    {
+    }
+
+    /**
+     * Before class.
+     * 
+     * @throws StatoolInfosException
+     *             the Juga exception
+     */
+    @BeforeClass
+    public static void beforeClass() throws StatoolInfosException
+    {
+        Configurator.initialize(new DefaultConfiguration());
+        Configurator.setRootLevel(Level.DEBUG);
+    }
+}

test/fr/devinsy/statoolinfos/metrics/httperror/HttpErrorLogsAnalyzerTest.java

@@ -32,9 +32,7 @@ import fr.devinsy.statoolinfos.core.Configuration;
 import fr.devinsy.statoolinfos.core.StatoolInfosException;
 import fr.devinsy.statoolinfos.metrics.PathCounters;
 import fr.devinsy.statoolinfos.metrics.Prober;
-import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLog;
 import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogAnalyzer;
-import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogIterator;
 import fr.devinsy.statoolinfos.metrics.httperrorlog.HttpErrorLogs;
 import fr.devinsy.statoolinfos.util.Files;
 import fr.devinsy.statoolinfos.util.FilesUtils;
@@ -42,53 +40,15 @@ import fr.devinsy.statoolinfos.util.FilesUtils;
 /**
  * The Class HttpErrorLogsTest.
  */
-public class HttpErrorLogsTest
+public class HttpErrorLogsAnalyzerTest
 {
-    private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpErrorLogsTest.class);
+    private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpErrorLogsAnalyzerTest.class);
 
-    /**
-     * Test 01.
-     *
-     * @throws Exception
-     *             the exception
-     */
     @Test
     public void test01() throws Exception
     {
         System.out.println(System.getProperty("user.dir"));
 
-        String source = "./test/fr/devinsy/statoolinfos/metrics/httperror/data/paste.libre-service.eu/paste*";
-        Files files = FilesUtils.searchByWildcard(source);
-        for (File file : files)
-        {
-            System.out.println(file);
-        }
-        HttpErrorLogs logs = new HttpErrorLogs(files);
-        HttpErrorLogIterator iterator = (HttpErrorLogIterator) logs.iterator();
-        while (iterator.hasNext())
-        {
-            HttpErrorLog log = iterator.next();
-            System.out.println(log.toStringLog());
-        }
-
-        // System.out.println(iterator.getLogCount());
-        // System.out.println(iterator.getFailedLogCount());
-
-        Assert.assertEquals(315, iterator.getLogCount());
-        Assert.assertEquals(0, iterator.getFailedLogCount());
-    }
-
-    /**
-     * Test 02.
-     *
-     * @throws Exception
-     *             the exception
-     */
-    @Test
-    public void test02() throws Exception
-    {
-        System.out.println(System.getProperty("user.dir"));
-
         String source = "./test/fr/devinsy/statoolinfos/metrics/httperror/data/paste.libre-service.eu/paste*";
         Files files = FilesUtils.searchByWildcard(source);
         for (File file : files)
@@ -112,14 +72,8 @@ public class HttpErrorLogsTest
         Assert.assertEquals(25, counters.getCount("metrics.http.errors", "2023-08"));
     }
 
-    /**
-     * Test 03.
-     *
-     * @throws Exception
-     *             the exception
-     */
     @Test
-    public void test03() throws Exception
+    public void test02() throws Exception
     {
         System.out.println(System.getProperty("user.dir"));
 
@@ -163,7 +117,7 @@ public class HttpErrorLogsTest
     }
 
     @Test
-    public void test04() throws Exception
+    public void test03() throws Exception
     {
         System.out.println(System.getProperty("user.dir"));
 

test/fr/devinsy/statoolinfos/metrics/httperror/data/paste.libre-service.eu/probe.conf

@@ -0,0 +1,8 @@
+# [Configuration] conf + federation + services + machines + metrics
+conf.class=service
+conf.protocol=StatoolInfos-0.3.0
+
+conf.probe.types=HttpErrorLog
+conf.probe.target=/tmp/t.metrics
+
+conf.probe.httperrorlog.file= test/fr/devinsy/statoolinfos/metrics/httperror/data/paste.libre-service.eu/paste*