From d4d1b6038cd1054bd93bcdc376944b92d8167b8c Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Sun, 25 Apr 2021 02:21:38 +0200 Subject: [PATCH] Refactored code. Removed concate use for anonymize. --- src/fr/devinsy/logar/app/Logar.java | 49 +++++-------------- .../logar/app/anonymizer/Anonymizer.java | 27 ++++------ src/org/april/logar/cli/LogarCLI.java | 14 +++--- src/org/april/logar/util/FilesUtils.java | 6 +-- 4 files changed, 31 insertions(+), 65 deletions(-) diff --git a/src/fr/devinsy/logar/app/Logar.java b/src/fr/devinsy/logar/app/Logar.java index 9da78c3..fc7774b 100644 --- a/src/fr/devinsy/logar/app/Logar.java +++ b/src/fr/devinsy/logar/app/Logar.java @@ -93,7 +93,7 @@ public final class Logar anonymizer.loadMapTable(mapFile); System.out.println("Table size=" + anonymizer.getMapTable().size()); - Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeContaining("-anon.log"); + Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log"); logger.info("file count={}", files.size()); for (File file : files) { @@ -338,15 +338,7 @@ public final class Logar else { System.out.println("== Check parse log for [" + file.getName() + "]"); - boolean isAccessFile; - if (file.getName().contains("access")) - { - isAccessFile = true; - } - else - { - isAccessFile = false; - } + boolean isAccessFile = file.getName().contains("access"); int lineCount = 0; int badLineCount = 0; @@ -403,7 +395,7 @@ public final class Logar */ public static void checkLogFiles(final File source) { - Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName(); + Files files = FilesUtils.search(source, LOGFILE_PATTERN).sortByName(); for (File file : files) { @@ -420,11 +412,11 @@ public final class Logar */ public static void checkSort(final File source) throws IOException { - Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName(); + Files files = FilesUtils.search(source, LOGFILE_PATTERN).sortByName(); for (File file : files) { - checkSortOfFile(file); + checkSortFile(file); } } @@ -435,7 +427,7 @@ public final class Logar * the source * @throws IOException */ - public static void checkSortOfFile(final File file) throws IOException + public static void checkSortFile(final File file) throws IOException { if (file == null) { @@ -447,17 +439,8 @@ public final class Logar } else { - boolean isAccessFile; - if (file.getName().contains("access")) - { - isAccessFile = true; - } - else - { - isAccessFile = false; - } - System.out.println("== Check sort for [" + file.getName() + "]"); + boolean isAccessFile = file.getName().contains("access"); LocalDateTime currentDate = null; int lineCount = 0; int badLineCount = 0; @@ -501,7 +484,7 @@ public final class Logar */ public static void sort(final File source) throws IOException { - Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName(); + Files files = FilesUtils.searchEndingWith(source, LOGFILE_PATTERN).removeHidden().sortByName(); for (File file : files) { @@ -518,11 +501,11 @@ public final class Logar */ public static void testConcate(final File source) { - Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName(); + Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log"); for (File file : files) { - testConcateOnFile(file); + testConcateFile(file); } } @@ -532,7 +515,7 @@ public final class Logar * @param source * the source */ - public static void testConcateOnFile(final File file) + public static void testConcateFile(final File file) { if (file == null) { @@ -545,15 +528,7 @@ public final class Logar else { System.out.println("== Test concate log for [" + file.getName() + "]"); - boolean isAccessFile; - if (file.getName().contains("access")) - { - isAccessFile = true; - } - else - { - isAccessFile = false; - } + boolean isAccessFile = file.getName().contains("access"); int lineCount = 0; int badLineCount = 0; diff --git a/src/fr/devinsy/logar/app/anonymizer/Anonymizer.java b/src/fr/devinsy/logar/app/anonymizer/Anonymizer.java index 010f27e..319db51 100644 --- a/src/fr/devinsy/logar/app/anonymizer/Anonymizer.java +++ b/src/fr/devinsy/logar/app/anonymizer/Anonymizer.java @@ -22,8 +22,6 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.time.format.DateTimeParseException; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -175,26 +173,19 @@ public final class Anonymizer { Log result; - result = new Log(log); + // + String anonIp = this.map.anonymizeIp(log.getIp()); + String line = log.getLine().replace(log.getIp(), anonIp); - result.setIp(this.map.anonymizeIp(log.getIp())); - result.setUser(this.map.anonymizeUser(log.getUser())); - - // Anonymize ip. - result.setRequest(result.getRequest().replace(result.getIp(), result.getIp())); - result.setReferer(result.getReferer().replace(result.getIp(), result.getIp())); - - // Anonymize user. - if (!log.getUser().equals("-")) + // + if (!StringUtils.equals(log.getUser(), "-")) { - // URLEncode replaces ' ' with '+' so bad for us. - String userInUrl = URLEncoder.encode(log.getUser(), StandardCharsets.UTF_8).replace("+", "%20"); - - result.setRequest(result.getRequest().replace(userInUrl, result.getUser())); - result.setReferer(result.getReferer().replace(userInUrl, result.getUser())); + String anonUser = this.map.anonymizeUser(log.getUser()); + line.replaceFirst(" " + log.getUser(), anonUser); } - result.concateAccessLog(); + // + result = new Log(line, log.getDatetime()); // return result; diff --git a/src/org/april/logar/cli/LogarCLI.java b/src/org/april/logar/cli/LogarCLI.java index d08117b..3d0003b 100644 --- a/src/org/april/logar/cli/LogarCLI.java +++ b/src/org/april/logar/cli/LogarCLI.java @@ -55,13 +55,13 @@ public final class LogarCLI message.appendln("Usage:"); message.appendln(" logar [ -h | -help | --help ]"); message.appendln(" logar [ -v | -version | --version ]"); - message.appendln(" logar anonymize fileordirectory [maptable] anonymize ip and login"); - message.appendln(" logar archive source target archive previous month"); - message.appendln(" logar check fileordirectory check line format in log files"); - message.appendln(" logar checksort fileordirectory check sort of an access log file"); - message.appendln(" logar sort fileordirectory sort log files by datetime"); - message.appendln(" logar testarchive source test archive"); - message.appendln(" logar testconcate fileordirectory test concate of log line"); + message.appendln(" logar anonymize fileordirectory [mapfile] anonymize ip and user"); + message.appendln(" logar archive source target archive previous month from /var/log/nginx/ tree"); + message.appendln(" logar check fileordirectory check line format in log file"); + message.appendln(" logar checksort fileordirectory check sort in log file"); + message.appendln(" logar sort fileordirectory sort log files by datetime"); + message.appendln(" logar testarchive source test archive without writing"); + message.appendln(" logar testconcate fileordirectory test line concate in log file"); logger.info(message.toString()); } diff --git a/src/org/april/logar/util/FilesUtils.java b/src/org/april/logar/util/FilesUtils.java index 8389aff..6d416fc 100644 --- a/src/org/april/logar/util/FilesUtils.java +++ b/src/org/april/logar/util/FilesUtils.java @@ -83,7 +83,7 @@ public class FilesUtils * the extensions * @return the files */ - public static Files searchFileRecursively(final File source, final String... extensions) + public static Files searchEndingWith(final File source, final String... extensions) { Files result; @@ -92,7 +92,7 @@ public class FilesUtils Files full = listRecursively(source); for (File file : full) { - if ((file.isFile()) && (StringUtils.endsWithAny(file.getName(), extensions))) + if (StringUtils.endsWithAny(file.getName(), extensions)) { result.add(file); } @@ -111,7 +111,7 @@ public class FilesUtils * the regex * @return the files */ - public static Files searchRecursively(final File source, final String regex) + public static Files search(final File source, final String regex) { Files result;