statool/logar
statool/logar
1
0
Fork 0
Code Issues Pull requests Releases Activity

Refactored code. Removed concate use for anonymize.

Browse source
This commit is contained in:
Christian P. MOMON 2021-04-25 02:21:38 +02:00
parent 1e46e07c4b
commit d4d1b6038c
4 changed files with 31 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
src/fr/devinsy/logar/app/Logar.java
View file

@ -93,7 +93,7 @@ public final class Logar
anonymizer.loadMapTable(mapFile);
System.out.println("Table size=" + anonymizer.getMapTable().size());
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeContaining("-anon.log");
Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log");
logger.info("file count={}", files.size());
for (File file : files)
{
@ -338,15 +338,7 @@ public final class Logar
else
{
System.out.println("== Check parse log for [" + file.getName() + "]");
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
boolean isAccessFile = file.getName().contains("access");
int lineCount = 0;
int badLineCount = 0;
@ -403,7 +395,7 @@ public final class Logar
*/
public static void checkLogFiles(final File source)
{
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
Files files = FilesUtils.search(source, LOGFILE_PATTERN).sortByName();
for (File file : files)
{
@ -420,11 +412,11 @@ public final class Logar
*/
public static void checkSort(final File source) throws IOException
{
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
Files files = FilesUtils.search(source, LOGFILE_PATTERN).sortByName();
for (File file : files)
{
checkSortOfFile(file);
checkSortFile(file);
}
}
@ -435,7 +427,7 @@ public final class Logar
* the source
* @throws IOException
*/
public static void checkSortOfFile(final File file) throws IOException
public static void checkSortFile(final File file) throws IOException
{
if (file == null)
{
@ -447,17 +439,8 @@ public final class Logar
}
else
{
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
System.out.println("== Check sort for [" + file.getName() + "]");
boolean isAccessFile = file.getName().contains("access");
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
@ -501,7 +484,7 @@ public final class Logar
*/
public static void sort(final File source) throws IOException
{
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName();
Files files = FilesUtils.searchEndingWith(source, LOGFILE_PATTERN).removeHidden().sortByName();
for (File file : files)
{
@ -518,11 +501,11 @@ public final class Logar
*/
public static void testConcate(final File source)
{
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log");
for (File file : files)
{
testConcateOnFile(file);
testConcateFile(file);
}
}
@ -532,7 +515,7 @@ public final class Logar
* @param source
* the source
*/
public static void testConcateOnFile(final File file)
public static void testConcateFile(final File file)
{
if (file == null)
{
@ -545,15 +528,7 @@ public final class Logar
else
{
System.out.println("== Test concate log for [" + file.getName() + "]");
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
boolean isAccessFile = file.getName().contains("access");
int lineCount = 0;
int badLineCount = 0;

27
src/fr/devinsy/logar/app/anonymizer/Anonymizer.java
View file

@ -22,8 +22,6 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.format.DateTimeParseException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -175,26 +173,19 @@ public final class Anonymizer
{
Log result;
result = new Log(log);
//
String anonIp = this.map.anonymizeIp(log.getIp());
String line = log.getLine().replace(log.getIp(), anonIp);
result.setIp(this.map.anonymizeIp(log.getIp()));
result.setUser(this.map.anonymizeUser(log.getUser()));
// Anonymize ip.
result.setRequest(result.getRequest().replace(result.getIp(), result.getIp()));
result.setReferer(result.getReferer().replace(result.getIp(), result.getIp()));
// Anonymize user.
if (!log.getUser().equals("-"))
//
if (!StringUtils.equals(log.getUser(), "-"))
{
// URLEncode replaces ' ' with '+' so bad for us.
String userInUrl = URLEncoder.encode(log.getUser(), StandardCharsets.UTF_8).replace("+", "%20");
result.setRequest(result.getRequest().replace(userInUrl, result.getUser()));
result.setReferer(result.getReferer().replace(userInUrl, result.getUser()));
String anonUser = this.map.anonymizeUser(log.getUser());
line.replaceFirst(" " + log.getUser(), anonUser);
}
result.concateAccessLog();
//
result = new Log(line, log.getDatetime());
//
return result;

14
src/org/april/logar/cli/LogarCLI.java
View file

@ -55,13 +55,13 @@ public final class LogarCLI
message.appendln("Usage:");
message.appendln(" logar [ -h | -help | --help ]");
message.appendln(" logar [ -v | -version | --version ]");
message.appendln(" logar anonymize fileordirectory [maptable] anonymize ip and login");
message.appendln(" logar archive source target archive previous month");
message.appendln(" logar check fileordirectory check line format in log files");
message.appendln(" logar checksort fileordirectory check sort of an access log file");
message.appendln(" logar sort fileordirectory sort log files by datetime");
message.appendln(" logar testarchive source test archive");
message.appendln(" logar testconcate fileordirectory test concate of log line");
message.appendln(" logar anonymize fileordirectory [mapfile] anonymize ip and user");
message.appendln(" logar archive source target archive previous month from /var/log/nginx/ tree");
message.appendln(" logar check fileordirectory check line format in log file");
message.appendln(" logar checksort fileordirectory check sort in log file");
message.appendln(" logar sort fileordirectory sort log files by datetime");
message.appendln(" logar testarchive source test archive without writing");
message.appendln(" logar testconcate fileordirectory test line concate in log file");
logger.info(message.toString());
}

6
src/org/april/logar/util/FilesUtils.java
View file

@ -83,7 +83,7 @@ public class FilesUtils
* the extensions
* @return the files
*/
public static Files searchFileRecursively(final File source, final String... extensions)
public static Files searchEndingWith(final File source, final String... extensions)
{
Files result;
@ -92,7 +92,7 @@ public class FilesUtils
Files full = listRecursively(source);
for (File file : full)
{
if ((file.isFile()) && (StringUtils.endsWithAny(file.getName(), extensions)))
if (StringUtils.endsWithAny(file.getName(), extensions))
{
result.add(file);
}
@ -111,7 +111,7 @@ public class FilesUtils
* the regex
* @return the files
*/
public static Files searchRecursively(final File source, final String regex)
public static Files search(final File source, final String regex)
{
Files result;