Refactored code. Improved anonymization.

src/fr/devinsy/logar/app/DryOption.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of Logar, simple tool to manage http log files.
+ * 
+ * Logar is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * Logar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Logar.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.logar.app;
+
+/**
+ * The Enum DryOption.
+ */
+public enum DryOption
+{
+    ON,
+    OFF;
+
+    /**
+     * Checks if is off.
+     *
+     * @return true, if is off
+     */
+    public boolean isOff()
+    {
+        boolean result;
+
+        result = (this == OFF);
+
+        //
+        return result;
+    }
+
+    /**
+     * Checks if is on.
+     *
+     * @return true, if is on
+     */
+    public boolean isOn()
+    {
+        boolean result;
+
+        result = (this == ON);
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/logar/app/Logar.java

@@ -24,11 +24,7 @@ import java.io.IOException;
 import java.io.PrintWriter;
 import java.time.LocalDateTime;
 import java.time.YearMonth;
-import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
-import java.util.Locale;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.zip.GZIPOutputStream;
 
 import org.apache.commons.io.IOUtils;
@@ -41,7 +37,8 @@ import org.slf4j.LoggerFactory;
 
 import fr.devinsy.logar.app.anonymizer.Anonymizer;
 import fr.devinsy.logar.app.log.Log;
-import fr.devinsy.logar.app.log.LogUtils;
+import fr.devinsy.logar.app.log.LogFile;
+import fr.devinsy.logar.app.log.LogParser;
 
 /**
  * The Class Logar.
@@ -50,8 +47,7 @@ public final class Logar
 {
     private static Logger logger = LoggerFactory.getLogger(Logar.class);
 
-    public static Pattern nginxAccessLogLinePattern = Pattern.compile("^\\S+ - [^\\[]+ \\[(?<time>[^\\]]+)\\] .*$");
+    public static String LOGFILE_PATTERN = "^.+\\.(log|log\\.\\d+|log\\.gz|log\\.gz\\.\\d+)";
-    public static Pattern nginxErrorLogLinePattern = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
 
     /**
      * Instantiates a new log tool.
@@ -121,10 +117,10 @@ public final class Logar
      *            the target
      * @throws IOException
      */
-    public static void archive(final File source, final File target) throws IOException
+    public static void archive(final File source, final File target, final DryOption dry) throws IOException
     {
-        archiveAccessFiles(source, target);
+        archiveAccessFiles(source, target, dry);
-        archiveErrorFiles(source, target);
+        archiveErrorFiles(source, target, dry);
     }
 
     /**
@@ -134,7 +130,7 @@ public final class Logar
      *            the source
      * @throws IOException
      */
-    public static void archiveAccessFiles(final File source, final File target) throws IOException
+    public static void archiveAccessFiles(final File source, final File target, final DryOption dry) throws IOException
     {
         if (source == null)
         {
@@ -153,14 +149,19 @@ public final class Logar
             YearMonth targetYearMonth = YearMonth.now().minusMonths(1);
             Stats counter = new Stats();
 
-            for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName())
+            for (File directory : Files.of(source).removeHidden().keepDirectoryType().sortByName())
             {
                 String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString());
                 File targetDirectory = new File(target, directory.getName());
-                targetDirectory.mkdirs();
                 File targetFile = new File(targetDirectory, targetFileName);
                 logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
-                PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
+
+                PrintWriter out = null;
+                if (dry.isOff())
+                {
+                    targetDirectory.mkdirs();
+                    out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
+                }
 
                 for (File file : Files.of(directory).sortByName())
                 {
@@ -177,12 +178,15 @@ public final class Logar
 
                                 try
                                 {
-                                    Log log = LogUtils.parseAccessLog(line);
+                                    Log log = LogParser.parseAccessLog(line);
                                     counter.incSuccessLineCount();
 
                                     if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
                                     {
-                                        out.println(line);
+                                        if (dry.isOff())
+                                        {
+                                            out.println(line);
+                                        }
                                     }
                                 }
                                 catch (IllegalArgumentException exception)
@@ -226,7 +230,7 @@ public final class Logar
      * @throws IOException
      *             Signals that an I/O exception has occurred.
      */
-    public static void archiveErrorFiles(final File source, final File target) throws IOException
+    public static void archiveErrorFiles(final File source, final File target, final DryOption dry) throws IOException
     {
         if (source == null)
         {
@@ -246,14 +250,18 @@ public final class Logar
             Stats counter = new Stats();
             counter.start();
 
-            for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName())
+            for (File directory : Files.of(source).removeHidden().keepDirectoryType().sortByName())
             {
                 String targetFileName = String.format("%s-error-%s.log.gz", directory.getName(), targetYearMonth.toString());
                 File targetDirectory = new File(target, directory.getName());
-                targetDirectory.mkdirs();
                 File targetFile = new File(targetDirectory, targetFileName);
                 logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
-                PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
+                PrintWriter out = null;
+                if (dry.isOff())
+                {
+                    targetDirectory.mkdirs();
+                    out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
+                }
 
                 for (File file : Files.of(directory).sortByName())
                 {
@@ -270,12 +278,15 @@ public final class Logar
 
                                 try
                                 {
-                                    Log log = LogUtils.parseErrorLog(line);
+                                    Log log = LogParser.parseErrorLog(line);
                                     counter.incSuccessLineCount();
 
                                     if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
                                     {
-                                        out.println(line);
+                                        if (dry.isOff())
+                                        {
+                                            out.println(line);
+                                        }
                                     }
                                 }
                                 catch (IllegalArgumentException exception)
@@ -326,7 +337,7 @@ public final class Logar
         }
         else
         {
-            System.out.println("== Check access log for [" + file.getName() + "]");
+            System.out.println("== Check parse log for [" + file.getName() + "]");
             boolean isAccessFile;
             if (file.getName().contains("access"))
             {
@@ -351,11 +362,11 @@ public final class Logar
                     {
                         if (isAccessFile)
                         {
-                            LogUtils.parseAccessLog(line).getDatetime();
+                            LogParser.parseAccessLog(line).getDatetime();
                         }
                         else
                         {
-                            LogUtils.parseErrorLog(line).getDatetime();
+                            LogParser.parseErrorLog(line).getDatetime();
                         }
                     }
                     catch (IllegalArgumentException exception)
@@ -392,7 +403,7 @@ public final class Logar
      */
     public static void checkLogFiles(final File source)
     {
-        Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz", ".1", ".2", ".3").removeHidden().sortByName();
+        Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
 
         for (File file : files)
         {
@@ -400,6 +411,23 @@ public final class Logar
         }
     }
 
+    /**
+     * Check sort.
+     *
+     * @param source
+     *            the source
+     * @throws IOException
+     */
+    public static void checkSort(final File source) throws IOException
+    {
+        Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
+
+        for (File file : files)
+        {
+            checkSortOfFile(file);
+        }
+    }
+
     /**
      * Check sort for access files.
      *
@@ -407,7 +435,7 @@ public final class Logar
      *            the source
      * @throws IOException
      */
-    public static void checkSort(final File file) throws IOException
+    public static void checkSortOfFile(final File file) throws IOException
     {
         if (file == null)
         {
@@ -441,11 +469,11 @@ public final class Logar
                 LocalDateTime date;
                 if (isAccessFile)
                 {
-                    date = LogUtils.parseAccessLog(line).getDatetime();
+                    date = LogParser.parseAccessLog(line).getDatetime();
                 }
                 else
                 {
-                    date = LogUtils.parseErrorLog(line).getDatetime();
+                    date = LogParser.parseErrorLog(line).getDatetime();
                 }
 
                 if ((currentDate != null) && (date.isBefore(currentDate)))
@@ -463,23 +491,6 @@ public final class Logar
         }
     }
 
-    /**
-     * Check sort.
-     *
-     * @param source
-     *            the source
-     * @throws IOException
-     */
-    public static void checkSorts(final File source) throws IOException
-    {
-        Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName();
-
-        for (File file : files)
-        {
-            checkSort(file);
-        }
-    }
-
     /**
      * Sort.
      *
@@ -495,189 +506,113 @@ public final class Logar
         for (File file : files)
         {
             System.out.println("== Sort for [" + file.getName() + "]");
-            LogUtils.sortLogFile(file);
+            LogFile.sortLogFile(file);
         }
     }
 
     /**
-     * Test parsing.
+     * Check concate.
      *
      * @param source
      *            the source
      */
-    public static void testParsing(final File source)
+    public static void testConcate(final File source)
     {
-        testParsingAccessFiles(source);
+        Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
-        testParsingErrorFiles(source);
+
+        for (File file : files)
+        {
+            testConcateOnFile(file);
+        }
     }
 
     /**
-     * Test parsing access files.
+     * Check concate on file.
      *
      * @param source
      *            the source
      */
-    public static void testParsingAccessFiles(final File source)
+    public static void testConcateOnFile(final File file)
     {
-        if (source == null)
+        if (file == null)
         {
-            throw new IllegalArgumentException("Null parameter [source]");
+            throw new IllegalArgumentException("Null parameter.");
         }
-        else if (!source.isDirectory())
+        else if (!file.isFile())
         {
-            throw new IllegalArgumentException("Source parameter is not a directory.");
+            throw new IllegalArgumentException("Parameter is not a file.");
         }
         else
         {
-            System.out.println("== Test access log for [" + source.getName() + "]");
+            System.out.println("== Test concate log for [" + file.getName() + "]");
-            Stats counter = new Stats();
+            boolean isAccessFile;
-
+            if (file.getName().contains("access"))
-            for (File directory : source.listFiles())
             {
-                if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
+                isAccessFile = true;
+            }
+            else
+            {
+                isAccessFile = false;
+            }
+
+            int lineCount = 0;
+            int badLineCount = 0;
+            try
+            {
+                LineIterator iterator = new LineIterator(file);
+                while (iterator.hasNext())
                 {
-                    logger.info("== {}", directory.getName());
+                    String line = iterator.next();
+                    lineCount += 1;
 
-                    for (File file : directory.listFiles())
+                    try
                     {
-                        if ((!file.isDirectory()) && (file.getName().contains("access")))
+                        Log source;
+                        Log target;
+                        if (isAccessFile)
                         {
-                            // logger.info(file.getName());
+                            source = LogParser.parseAccessLog(line);
-
+                            target = new Log(source);
-                            try
+                            target.concateAccessLog();
-                            {
-                                LineIterator iterator = new LineIterator(file);
-                                while (iterator.hasNext())
-                                {
-                                    String line = iterator.next();
-                                    counter.incLineCount();
-
-                                    Matcher matcher = nginxAccessLogLinePattern.matcher(line);
-                                    if (matcher.matches())
-                                    {
-                                        String value = matcher.group("time");
-                                        try
-                                        {
-                                            LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
-                                            counter.incSuccessLineCount();
-                                        }
-                                        catch (DateTimeParseException exception)
-                                        {
-                                            System.err.println("Date format error with [" + line + "]");
-                                            counter.incErrorLineCount();
-                                        }
-                                    }
-                                    else
-                                    {
-                                        System.err.println("Not matching line [" + line + "]");
-                                        counter.incErrorLineCount();
-                                    }
-                                }
-
-                                counter.incSuccessFileCount();
-                            }
-                            catch (IOException exception)
-                            {
-                                System.err.println("Error with file [" + file.getAbsolutePath() + "]");
-                                exception.printStackTrace();
-                                counter.incErrorFileCount();
-                            }
-                            finally
-                            {
-                                counter.incFileCount();
-                            }
                         }
+                        else
+                        {
+                            source = LogParser.parseErrorLog(line);
+                            target = new Log(source);
+                            target.concateErrorLog();
+                        }
+
+                        if (!StringUtils.equals(source.getLine(), target.getLine()))
+                        {
+                            System.out.println("Bad concat (1): " + source);
+                            System.out.println("Bad concat (2): " + target);
+                            badLineCount += 1;
+                        }
+
+                    }
+                    catch (IllegalArgumentException exception)
+                    {
+                        System.out.println("Bad format line: " + line);
+                        badLineCount += 1;
+                        exception.printStackTrace();
+                    }
+                    catch (DateTimeParseException exception)
+                    {
+                        System.out.println("Bad datetime format: " + line);
+                        badLineCount += 1;
                     }
                 }
             }
-
+            catch (IOException exception)
-            counter.stop();
-            System.out.println(counter.toString());
-        }
-    }
-
-    /**
-     * Test parsing error files.
-     *
-     * @param source
-     *            the source
-     */
-    public static void testParsingErrorFiles(final File source)
-    {
-        if (source == null)
-        {
-            throw new IllegalArgumentException("Null parameter [source]");
-        }
-        else if (!source.isDirectory())
-        {
-            throw new IllegalArgumentException("Source parameter is not a directory.");
-        }
-        else
-        {
-            System.out.println("== Test error log for [" + source.getName() + "]");
-            Stats counter = new Stats();
-
-            for (File directory : source.listFiles())
             {
-                if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
+                System.err.println("Error with file [" + file.getAbsolutePath() + "]");
-                {
+                exception.printStackTrace();
-                    logger.info("== {}", directory.getName());
-
-                    for (File file : directory.listFiles())
-                    {
-                        if ((!file.isDirectory()) && (file.getName().contains("error")))
-                        {
-                            // logger.info(file.getName());
-                            try
-                            {
-                                LineIterator iterator = new LineIterator(file);
-                                while (iterator.hasNext())
-                                {
-                                    String line = iterator.next();
-                                    counter.incLineCount();
-
-                                    Matcher matcher = nginxErrorLogLinePattern.matcher(line);
-                                    if (matcher.matches())
-                                    {
-                                        String value = matcher.group("time");
-                                        try
-                                        {
-                                            LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
-                                            counter.incSuccessLineCount();
-                                        }
-                                        catch (DateTimeParseException exception)
-                                        {
-                                            System.err.println("Date format problem with [" + line + "]");
-                                            counter.incErrorLineCount();
-                                        }
-                                    }
-                                    else
-                                    {
-                                        System.err.println("Not matching line [" + line + "]");
-                                        counter.incErrorLineCount();
-                                    }
-                                }
-
-                                counter.incSuccessFileCount();
-                            }
-                            catch (IOException exception)
-                            {
-                                System.err.println("Error with file [" + file.getAbsolutePath() + "]");
-                                exception.printStackTrace();
-                                counter.incErrorFileCount();
-                            }
-                            finally
-                            {
-                                counter.incFileCount();
-                            }
-                        }
-                    }
-                }
             }
 
-            System.out.println("=====================================================");
+            if (badLineCount > 0)
-            counter.stop();
+            {
-            System.out.println(counter.toString());
+                System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
+            }
         }
     }
 }

src/fr/devinsy/logar/app/anonymizer/Anonymizer.java

@@ -22,6 +22,8 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.time.format.DateTimeParseException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -34,7 +36,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import fr.devinsy.logar.app.log.Log;
-import fr.devinsy.logar.app.log.LogUtils;
+import fr.devinsy.logar.app.log.LogParser;
 
 /**
  * The Class Anonymizer.
@@ -120,7 +122,7 @@ public final class Anonymizer
                         Log anon;
                         if (isAccessFile)
                         {
-                            Log log = LogUtils.parseAccessLog(line);
+                            Log log = LogParser.parseAccessLog(line);
                             // logger.info("line={}", line);
                             // logger.info("log =[{}][{}][{}]", log.getIp(),
                             // log.getUser(), log.getDatetime());
@@ -132,7 +134,7 @@ public final class Anonymizer
                         }
                         else
                         {
-                            Log log = LogUtils.parseErrorLog(line);
+                            Log log = LogParser.parseErrorLog(line);
 
                             anon = anonymizeError(log);
                         }
@@ -173,16 +175,26 @@ public final class Anonymizer
     {
         Log result;
 
-        String anonIp = this.map.anonymizeIp(log.getIp());
+        result = new Log(log);
-        String anonUser = this.map.anonymizeUser(log.getUser());
 
-        String line = log.getLine().replace(log.getIp(), anonIp);
+        result.setIp(this.map.anonymizeIp(log.getIp()));
+        result.setUser(this.map.anonymizeUser(log.getUser()));
+
+        // Anonymize ip.
+        result.setRequest(result.getRequest().replace(result.getIp(), result.getIp()));
+        result.setReferer(result.getReferer().replace(result.getIp(), result.getIp()));
+
+        // Anonymize user.
         if (!log.getUser().equals("-"))
         {
-            line = line.replace(log.getUser(), anonUser);
+            // URLEncode replaces ' ' with '+' so bad for us.
+            String userInUrl = URLEncoder.encode(log.getUser(), StandardCharsets.UTF_8).replace("+", "%20");
+
+            result.setRequest(result.getRequest().replace(userInUrl, result.getUser()));
+            result.setReferer(result.getReferer().replace(userInUrl, result.getUser()));
         }
 
-        result = new Log(line, log.getDatetime(), anonIp, anonUser);
+        result.concateAccessLog();
 
         //
         return result;

src/fr/devinsy/logar/app/log/Log.java

@@ -33,13 +33,44 @@ public final class Log
     // Generic attributes.
     private String line;
     private LocalDateTime datetime;
+    private String datetimeValue;
 
     // Specific access log attributes.
     private String ip;
     private String user;
+    private String request;
+    private String status;
+    private String bodyByteSent;
+    private String referer;
+    private String userAgent;
 
     // Specific error log attributes.
-    // private String message;
+    private String level;
+    private String message;
+
+    /**
+     * Instantiates a new log.
+     *
+     * @param log
+     *            the log
+     */
+    public Log(final Log log)
+    {
+        this.line = log.getLine();
+        this.datetime = log.getDatetime();
+        this.datetimeValue = log.getDatetimeValue();
+
+        this.ip = log.getIp();
+        this.user = log.getUser();
+        this.request = log.getRequest();
+        this.status = log.getStatus();
+        this.bodyByteSent = log.getBodyByteSent();
+        this.referer = log.getReferer();
+        this.userAgent = log.getUserAgent();
+
+        this.level = log.getLevel();
+        this.message = log.getMessage();
+    }
 
     /**
      * Instantiates a new log.
@@ -48,28 +79,39 @@ public final class Log
     {
         this.line = line;
         this.datetime = datetime;
+        this.datetimeValue = null;
+
         this.ip = null;
         this.user = null;
+        this.request = null;
+        this.referer = null;
+        this.userAgent = null;
+
+        this.level = null;
+        this.message = null;
     }
 
     /**
-     * Instantiates a new log.
+     * Concate access log.
-     *
-     * @param line
-     *            the line
-     * @param datetime
-     *            the datetime
-     * @param ip
-     *            the ip
-     * @param user
-     *            the login
      */
-    public Log(final String line, final LocalDateTime datetime, final String ip, final String user)
+    public void concateAccessLog()
     {
-        this.line = line;
+        this.line = String.format("%s - %s [%s] \"%s\" %s %s \"%s\" \"%s\"",
-        this.datetime = datetime;
+                this.ip, this.user, this.datetimeValue, this.request, this.status, this.bodyByteSent, this.referer, this.userAgent);
-        this.ip = ip;
+    }
-        this.user = user;
+
+    /**
+     * Concate error log.
+     */
+    public void concateErrorLog()
+    {
+        this.line = String.format("%s [%s] %s",
+                this.datetimeValue, this.level, this.message);
+    }
+
+    public String getBodyByteSent()
+    {
+        return this.bodyByteSent;
     }
 
     public LocalDateTime getDatetime()
@@ -77,21 +119,128 @@ public final class Log
         return this.datetime;
     }
 
+    public String getDatetimeValue()
+    {
+        return this.datetimeValue;
+    }
+
     public String getIp()
     {
         return this.ip;
     }
 
+    public String getLevel()
+    {
+        return this.level;
+    }
+
     public String getLine()
     {
         return this.line;
     }
 
+    public String getMessage()
+    {
+        return this.message;
+    }
+
+    public String getReferer()
+    {
+        return this.referer;
+    }
+
+    public String getRequest()
+    {
+        return this.request;
+    }
+
+    public String getStatus()
+    {
+        return this.status;
+    }
+
     public String getUser()
     {
         return this.user;
     }
 
+    public String getUserAgent()
+    {
+        return this.userAgent;
+    }
+
+    /**
+     * Reduce.
+     */
+    public void reduce()
+    {
+        this.datetimeValue = null;
+        this.request = null;
+        this.referer = null;
+        this.userAgent = null;
+        this.status = null;
+        this.bodyByteSent = null;
+
+        this.level = null;
+        this.message = null;
+    }
+
+    public void setBodyByteSent(final String bodyByteSent)
+    {
+        this.bodyByteSent = bodyByteSent;
+    }
+
+    public void setDatetimeValue(final String datetimeValue)
+    {
+        this.datetimeValue = datetimeValue;
+    }
+
+    public void setIp(final String ip)
+    {
+        this.ip = ip;
+    }
+
+    public void setLevel(final String level)
+    {
+        this.level = level;
+    }
+
+    public void setMessage(final String message)
+    {
+        this.message = message;
+    }
+
+    public void setReferer(final String referer)
+    {
+        this.referer = referer;
+    }
+
+    public void setRequest(final String request)
+    {
+        this.request = request;
+    }
+
+    public void setStatus(final String status)
+    {
+        this.status = status;
+    }
+
+    public void setUser(final String user)
+    {
+        this.user = user;
+    }
+
+    /**
+     * Sets the user agent.
+     *
+     * @param userAgent
+     *            the new user agent
+     */
+    public void setUserAgent(final String userAgent)
+    {
+        this.userAgent = userAgent;
+    }
+
     /* (non-Javadoc)
      * @see java.lang.Object#toString()
      */

src/fr/devinsy/logar/app/log/LogFile.java

@@ -0,0 +1,256 @@
+/*
+ * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of Logar, simple tool to manage http log files.
+ * 
+ * Logar is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * Logar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Logar.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.logar.app.log;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.april.logar.util.LineIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import fr.devinsy.cmdexec.CmdExecException;
+import fr.devinsy.cmdexec.CmdExecUtils;
+
+/**
+ * The Class LogFile.
+ */
+public final class LogFile
+{
+    private static Logger logger = LoggerFactory.getLogger(LogFile.class);
+
+    /**
+     * Instantiates a new log file.
+     */
+    private LogFile()
+    {
+    }
+
+    /**
+     * Load access log.
+     *
+     * @param file
+     *            the file
+     * @return the logs
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public static Logs loadAccessLog(final File file) throws IOException
+    {
+        Logs result;
+
+        result = loadAccessLog(file, LogMode.NORMAL);
+
+        return result;
+    }
+
+    /**
+     * Load access log.
+     *
+     * @param file
+     *            the file
+     * @param mode
+     *            the mode
+     * @return the logs
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public static Logs loadAccessLog(final File file, final LogMode mode) throws IOException
+    {
+        Logs result;
+
+        result = new Logs();
+
+        LineIterator iterator = new LineIterator(file);
+        while (iterator.hasNext())
+        {
+            String line = iterator.next();
+            Log log = LogParser.parseAccessLog(line);
+            if (mode == LogMode.REDUCED)
+            {
+                log.reduce();
+            }
+            result.add(log);
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Load error log.
+     *
+     * @param file
+     *            the file
+     * @return the logs
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public static Logs loadErrorLog(final File file) throws IOException
+    {
+        Logs result;
+
+        result = new Logs();
+
+        LineIterator iterator = new LineIterator(file);
+        while (iterator.hasNext())
+        {
+            String line = iterator.next();
+            Log log = LogParser.parseErrorLog(line);
+            result.add(log);
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Load log file.
+     *
+     * @param file
+     *            the file
+     * @return the logs
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public static Logs loadLogFile(final File file) throws IOException
+    {
+        Logs result;
+
+        result = loadLogFile(file, LogMode.NORMAL);
+
+        //
+        return result;
+    }
+
+    /**
+     * Load log file.
+     *
+     * @param file
+     *            the file
+     * @param mode
+     *            the mode
+     * @return the logs
+     * @throws IOException
+     *             Signals that an I/O exception has occurred.
+     */
+    public static Logs loadLogFile(final File file, final LogMode mode) throws IOException
+    {
+        Logs result;
+
+        if (file == null)
+        {
+            throw new IllegalArgumentException("Null parameter.");
+        }
+        else
+        {
+            if (file.getName().contains("access"))
+            {
+                result = loadAccessLog(file, mode);
+            }
+            else if (file.getName().contains("error"))
+            {
+                result = loadErrorLog(file);
+            }
+            else
+            {
+                throw new IllegalArgumentException("Bad named file (missing access or error).");
+            }
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Save access log.
+     *
+     * @param source
+     *            the source
+     * @throws IOException
+     * @throws FileNotFoundException
+     */
+    public static void saveLogs(final File target, final Logs logs) throws FileNotFoundException, IOException
+    {
+        PrintWriter out = null;
+        try
+        {
+            if (target.getName().endsWith(".gz"))
+            {
+                out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
+            }
+            else
+            {
+                out = new PrintWriter(new FileOutputStream(target));
+            }
+
+            for (Log log : logs)
+            {
+                out.println(log.getLine());
+            }
+        }
+        finally
+        {
+            IOUtils.closeQuietly(out);
+        }
+    }
+
+    /**
+     * Sort access log.
+     *
+     * @param target
+     *            the target
+     * @throws IOException
+     */
+    public static void sortLogFile(final File file) throws IOException
+    {
+        File workFile = new File(file.getParent(), file.getName() + ".tmp");
+
+        Logs logs = loadLogFile(file, LogMode.REDUCED);
+        logs.sortByDatetime();
+        saveLogs(workFile, logs);
+
+        File backup = new File(file.getParentFile(), file.getName() + ".bak");
+        if (file.renameTo(backup))
+        {
+            if (!workFile.renameTo(file))
+            {
+                backup.renameTo(file);
+            }
+        }
+
+        // Check.
+        try
+        {
+            String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum  \"");
+            System.out.print(out);
+            out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum  \"");
+            System.out.println(out);
+        }
+        catch (CmdExecException exception)
+        {
+            exception.printStackTrace();
+        }
+    }
+}

src/fr/devinsy/logar/app/log/LogMode.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of Logar, simple tool to manage http log files.
+ * 
+ * Logar is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * Logar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Logar.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.logar.app.log;
+
+/**
+ * The Enum LogMode.
+ */
+public enum LogMode
+{
+    NORMAL,
+    REDUCED;
+}

src/fr/devinsy/logar/app/log/LogParser.java

@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
+ * 
+ * This file is part of Logar, simple tool to manage http log files.
+ * 
+ * Logar is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ * 
+ * Logar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Logar.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package fr.devinsy.logar.app.log;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The Class LogParser.
+ */
+public final class LogParser
+{
+    private static Logger logger = LoggerFactory.getLogger(LogParser.class);
+
+    public static Pattern NGINX_ACCESSLOG_LINE_PATTERN = Pattern.compile(
+            "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<datetime>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
+
+    public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?<datetime>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
+
+    /**
+     * Instantiates a new log parser.
+     */
+    private LogParser()
+    {
+    }
+
+    /**
+     * From access log.
+     *
+     * @param line
+     *            the line
+     * @return the log
+     * 
+     *         log_format combined '$remote_addr - $remote_user [$time_local] '
+     *         '"$request" $status $body_bytes_sent ' '"$http_referer"
+     *         "$http_user_agent"';
+     */
+    public static Log parseAccessLog(final String line)
+    {
+        Log result;
+
+        try
+        {
+            Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
+            if (matcher.matches())
+            {
+                String dateTimeValue = matcher.group("datetime");
+                LocalDateTime dateTime = LocalDateTime.parse(dateTimeValue, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
+
+                result = new Log(line, dateTime);
+
+                result.setDatetimeValue(matcher.group("datetime"));
+                result.setIp(matcher.group("remoteAddress"));
+                result.setUser(matcher.group("remoteUser"));
+                result.setRequest(matcher.group("request"));
+                result.setStatus(matcher.group("status"));
+                result.setBodyByteSent(matcher.group("bodyBytesSent"));
+                result.setReferer(matcher.group("referer"));
+                result.setUserAgent(matcher.group("userAgent"));
+            }
+            else
+            {
+                throw new IllegalArgumentException("Bad line format: " + line);
+            }
+        }
+        catch (DateTimeParseException exception)
+        {
+            throw new IllegalArgumentException("Bad line format (date): " + line);
+        }
+
+        //
+        return result;
+    }
+
+    /**
+     * Parses the access log light.
+     *
+     * @param line
+     *            the line
+     * @return the log
+     */
+    public static Log parseAccessLogLight(final String line)
+    {
+        Log result;
+
+        result = parseAccessLog(line);
+        result.reduce();
+
+        //
+        return result;
+    }
+
+    /**
+     * From error log.
+     *
+     * @param line
+     *            the line
+     * @return the log
+     */
+    public static Log parseErrorLog(final String line)
+    {
+        Log result;
+
+        try
+        {
+            Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
+            if (matcher.matches())
+            {
+                String value = matcher.group("datetime");
+                LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
+
+                result = new Log(line, date);
+                result.setDatetimeValue(matcher.group("datetime"));
+                result.setLevel(matcher.group("level"));
+                result.setMessage(matcher.group("message"));
+            }
+            else
+            {
+                throw new IllegalArgumentException("Bad line format: " + line);
+            }
+        }
+        catch (DateTimeParseException exception)
+        {
+            throw new IllegalArgumentException("Bad line format (date): " + line);
+        }
+
+        //
+        return result;
+    }
+}

src/fr/devinsy/logar/app/log/LogUtils.java

@@ -18,278 +18,20 @@
  */
 package fr.devinsy.logar.app.log;
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.time.LocalDateTime;
-import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeParseException;
-import java.util.Locale;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPOutputStream;
-
-import org.apache.commons.io.IOUtils;
-import org.april.logar.util.LineIterator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import fr.devinsy.cmdexec.CmdExecException;
-import fr.devinsy.cmdexec.CmdExecUtils;
-
 /**
- * The Class NginxAccessLogParser.
+ * The Class LogUtils.
  */
 public final class LogUtils
 {
     private static Logger logger = LoggerFactory.getLogger(LogUtils.class);
 
-    public static Pattern nginxAccessLogLinePatternFull = Pattern.compile(
-            "^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
-
-    public static Pattern NGINX_ACCESSLOG_LINE_PATTERN = Pattern.compile("^(?<remoteAddress>[a-fA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] .*$");
-    public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
-
     /**
-     * Instantiates a new nginx access log parser.
+     * Instantiates a new log utils.
      */
     private LogUtils()
     {
     }
-
-    /**
-     * Load.
-     *
-     * @param file
-     *            the file
-     * @return the logs
-     * @throws IOException
-     */
-    public static Logs loadAccessLog(final File file) throws IOException
-    {
-        Logs result;
-
-        result = new Logs();
-
-        LineIterator iterator = new LineIterator(file);
-        while (iterator.hasNext())
-        {
-            String line = iterator.next();
-            Log log = parseAccessLog(line);
-            result.add(log);
-        }
-
-        //
-        return result;
-    }
-
-    /**
-     * Load error log.
-     *
-     * @param file
-     *            the file
-     * @return the logs
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
-     */
-    public static Logs loadErrorLog(final File file) throws IOException
-    {
-        Logs result;
-
-        result = new Logs();
-
-        LineIterator iterator = new LineIterator(file);
-        while (iterator.hasNext())
-        {
-            String line = iterator.next();
-            Log log = parseErrorLog(line);
-            result.add(log);
-        }
-
-        //
-        return result;
-    }
-
-    /**
-     * Load log file.
-     *
-     * @param file
-     *            the file
-     * @return the logs
-     * @throws IOException
-     *             Signals that an I/O exception has occurred.
-     */
-    public static Logs loadLogFile(final File file) throws IOException
-    {
-        Logs result;
-
-        if (file == null)
-        {
-            throw new IllegalArgumentException("Null parameter.");
-        }
-        else
-        {
-            if (file.getName().contains("access"))
-            {
-                result = loadAccessLog(file);
-            }
-            else if (file.getName().contains("error"))
-            {
-                result = loadErrorLog(file);
-            }
-            else
-            {
-                throw new IllegalArgumentException("Bad named file (missing access or error).");
-            }
-        }
-
-        //
-        return result;
-    }
-
-    /**
-     * From access log.
-     *
-     * @param line
-     *            the line
-     * @return the log
-     */
-    public static Log parseAccessLog(final String line)
-    {
-        Log result;
-
-        try
-        {
-            Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
-            if (matcher.matches())
-            {
-                String value = matcher.group("time");
-                LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
-
-                String ip = matcher.group("remoteAddress").trim();
-                String user = matcher.group("remoteUser").trim();
-
-                result = new Log(line, date, ip, user);
-            }
-            else
-            {
-                throw new IllegalArgumentException("Bad line format: " + line);
-            }
-        }
-        catch (DateTimeParseException exception)
-        {
-            throw new IllegalArgumentException("Bad line format (date): " + line);
-        }
-
-        //
-        return result;
-    }
-
-    /**
-     * From error log.
-     *
-     * @param line
-     *            the line
-     * @return the log
-     */
-    public static Log parseErrorLog(final String line)
-    {
-        Log result;
-
-        try
-        {
-            Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
-            if (matcher.matches())
-            {
-                String value = matcher.group("time");
-                LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
-
-                result = new Log(line, date);
-            }
-            else
-            {
-                throw new IllegalArgumentException("Bad line format: " + line);
-            }
-        }
-        catch (DateTimeParseException exception)
-        {
-            throw new IllegalArgumentException("Bad line format (date): " + line);
-        }
-
-        //
-        return result;
-    }
-
-    /**
-     * Save access log.
-     *
-     * @param source
-     *            the source
-     * @throws IOException
-     * @throws FileNotFoundException
-     */
-    public static void saveLogs(final File target, final Logs logs) throws FileNotFoundException, IOException
-    {
-        PrintWriter out = null;
-        try
-        {
-            if (target.getName().endsWith(".gz"))
-            {
-                out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
-            }
-            else
-            {
-                out = new PrintWriter(new FileOutputStream(target));
-            }
-
-            for (Log log : logs)
-            {
-                out.println(log.getLine());
-            }
-        }
-        finally
-        {
-            IOUtils.closeQuietly(out);
-        }
-    }
-
-    /**
-     * Sort access log.
-     *
-     * @param target
-     *            the target
-     * @throws IOException
-     */
-    public static void sortLogFile(final File file) throws IOException
-    {
-        File workFile = new File(file.getParent(), file.getName() + ".tmp");
-
-        Logs logs = loadLogFile(file);
-        logs.sortByDatetime();
-        saveLogs(workFile, logs);
-
-        File backup = new File(file.getParentFile(), file.getName() + ".bak");
-        if (file.renameTo(backup))
-        {
-            if (!workFile.renameTo(file))
-            {
-                backup.renameTo(file);
-            }
-        }
-
-        // Check.
-        try
-        {
-            String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum  \"");
-            System.out.print(out);
-            out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum  \"");
-            System.out.println(out);
-        }
-        catch (CmdExecException exception)
-        {
-            exception.printStackTrace();
-        }
-    }
 }

src/org/april/logar/cli/LogarCLI.java

@@ -25,6 +25,7 @@ import org.april.logar.util.BuildInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import fr.devinsy.logar.app.DryOption;
 import fr.devinsy.logar.app.Logar;
 import fr.devinsy.strings.StringList;
 
@@ -56,10 +57,11 @@ public final class LogarCLI
         message.appendln("    logar [ -v | -version | --version ]");
         message.appendln("    logar anonymize fileordirectory [maptable]    anonymize ip and login");
         message.appendln("    logar archive source target                   archive previous month");
-        message.appendln("    logar check     fileordirectory               census bad format line in log files");
+        message.appendln("    logar check        fileordirectory            check line format in log files");
-        message.appendln("    logar checksort fileordirectory               check sort of an access log file");
+        message.appendln("    logar checksort    fileordirectory            check sort of an access log file");
         message.appendln("    logar sort fileordirectory                    sort log files by datetime");
         message.appendln("    logar testarchive source                      test archive");
+        message.appendln("    logar testconcate fileordirectory             test concate of log line");
 
         logger.info(message.toString());
     }
@@ -201,7 +203,7 @@ public final class LogarCLI
                 File source = new File(args[1]);
                 File target = new File(args[2]);
 
-                Logar.archive(source, target);
+                Logar.archive(source, target, DryOption.OFF);
             }
             else if (isMatching(args, "check", "\\s*\\S+\\s*"))
             {
@@ -213,7 +215,7 @@ public final class LogarCLI
             {
                 File source = new File(args[1]);
 
-                Logar.checkSorts(source);
+                Logar.checkSort(source);
             }
             else if (isMatching(args, "sort", "\\s*\\S+\\s*"))
             {
@@ -221,11 +223,18 @@ public final class LogarCLI
 
                 Logar.sort(source);
             }
-            else if (isMatching(args, "testarchive", "\\s*\\S+\\s*"))
+            else if (isMatching(args, "testarchive", "\\s*\\S+\\s*", "\\s*\\S+\\s*"))
+            {
+                File source = new File(args[1]);
+                File target = new File(args[2]);
+
+                Logar.archive(source, target, DryOption.ON);
+            }
+            else if (isMatching(args, "testconcate", "\\s*\\S+\\s*"))
             {
                 File source = new File(args[1]);
 
-                Logar.testParsing(source);
+                Logar.testConcate(source);
             }
             else
             {

src/org/april/logar/util/Files.java

@@ -21,6 +21,7 @@ package org.april.logar.util;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.regex.Pattern;
 
 /**
  * The Class Files.
@@ -48,12 +49,42 @@ public class Files extends ArrayList<File>
         super(initialCapacity);
     }
 
+    /**
+     * Keep.
+     *
+     * @param regex
+     *            the regex
+     * @return the files
+     */
+    public Files keep(final String regex)
+    {
+        Files result;
+
+        Pattern pattern = Pattern.compile(regex);
+
+        Iterator<File> iterator = iterator();
+        while (iterator.hasNext())
+        {
+            File file = iterator.next();
+
+            if (!pattern.matcher(file.getName()).matches())
+            {
+                iterator.remove();
+            }
+        }
+
+        result = this;
+
+        //
+        return result;
+    }
+
     /**
      * Keep directories.
      *
      * @return the files
      */
-    public Files keepDirectoriesOnly()
+    public Files keepDirectoryType()
     {
         Files result;
 
@@ -73,6 +104,31 @@ public class Files extends ArrayList<File>
         return result;
     }
 
+    /**
+     * Keep file type.
+     *
+     * @return the files
+     */
+    public Files keepFileType()
+    {
+        Files result;
+
+        Iterator<File> iterator = iterator();
+        while (iterator.hasNext())
+        {
+            File file = iterator.next();
+            if (!file.isFile())
+            {
+                iterator.remove();
+            }
+        }
+
+        result = this;
+
+        //
+        return result;
+    }
+
     /**
      * Removes the containing.
      *

src/org/april/logar/util/FilesUtils.java

@@ -19,6 +19,7 @@
 package org.april.logar.util;
 
 import java.io.File;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
 
@@ -100,4 +101,34 @@ public class FilesUtils
         //
         return result;
     }
+
+    /**
+     * Search recursively.
+     *
+     * @param source
+     *            the source
+     * @param regex
+     *            the regex
+     * @return the files
+     */
+    public static Files searchRecursively(final File source, final String regex)
+    {
+        Files result;
+
+        result = new Files();
+
+        Pattern pattern = Pattern.compile(regex);
+
+        Files full = listRecursively(source);
+        for (File file : full)
+        {
+            if (pattern.matcher(file.getName()).matches())
+            {
+                result.add(file);
+            }
+        }
+
+        //
+        return result;
+    }
 }