statool/logar
statool/logar
1
0
Fork 0
Code Issues Pull requests Releases Activity

Refactored code. Improved anonymization.

Browse source
This commit is contained in:
Christian P. MOMON 2021-04-24 16:48:30 +02:00
parent c3a6fc9841
commit 1e46e07c4b
11 changed files with 914 additions and 485 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
src/fr/devinsy/logar/app/DryOption.java Normal file
View file

@ -0,0 +1,58 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app;
/**
* The Enum DryOption.
*/
public enum DryOption
{
ON,
OFF;
/**
* Checks if is off.
*
* @return true, if is off
*/
public boolean isOff()
{
boolean result;
result = (this == OFF);
//
return result;
}
/**
* Checks if is on.
*
* @return true, if is on
*/
public boolean isOn()
{
boolean result;
result = (this == ON);
//
return result;
}
}

293
src/fr/devinsy/logar/app/Logar.java
View file

@ -24,11 +24,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.time.LocalDateTime;
import java.time.YearMonth;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;
@ -41,7 +37,8 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.anonymizer.Anonymizer;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogUtils;
import fr.devinsy.logar.app.log.LogFile;
import fr.devinsy.logar.app.log.LogParser;
/**
* The Class Logar.
@ -50,8 +47,7 @@ public final class Logar
{
private static Logger logger = LoggerFactory.getLogger(Logar.class);
public static Pattern nginxAccessLogLinePattern = Pattern.compile("^\\S+ - [^\\[]+ \\[(?<time>[^\\]]+)\\] .*$");
public static Pattern nginxErrorLogLinePattern = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s.*$");
public static String LOGFILE_PATTERN = "^.+\\.(log|log\\.\\d+|log\\.gz|log\\.gz\\.\\d+)";
/**
* Instantiates a new log tool.
@ -121,10 +117,10 @@ public final class Logar
* the target
* @throws IOException
*/
public static void archive(final File source, final File target) throws IOException
public static void archive(final File source, final File target, final DryOption dry) throws IOException
{
archiveAccessFiles(source, target);
archiveErrorFiles(source, target);
archiveAccessFiles(source, target, dry);
archiveErrorFiles(source, target, dry);
}
/**
@ -134,7 +130,7 @@ public final class Logar
* the source
* @throws IOException
*/
public static void archiveAccessFiles(final File source, final File target) throws IOException
public static void archiveAccessFiles(final File source, final File target, final DryOption dry) throws IOException
{
if (source == null)
{
@ -153,14 +149,19 @@ public final class Logar
YearMonth targetYearMonth = YearMonth.now().minusMonths(1);
Stats counter = new Stats();
for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName())
for (File directory : Files.of(source).removeHidden().keepDirectoryType().sortByName())
{
String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString());
File targetDirectory = new File(target, directory.getName());
targetDirectory.mkdirs();
File targetFile = new File(targetDirectory, targetFileName);
logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
PrintWriter out = null;
if (dry.isOff())
{
targetDirectory.mkdirs();
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
}
for (File file : Files.of(directory).sortByName())
{
@ -177,14 +178,17 @@ public final class Logar
try
{
Log log = LogUtils.parseAccessLog(line);
Log log = LogParser.parseAccessLog(line);
counter.incSuccessLineCount();
if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
{
if (dry.isOff())
{
out.println(line);
}
}
}
catch (IllegalArgumentException exception)
{
System.err.println("Bad line format [" + line + "]");
@ -226,7 +230,7 @@ public final class Logar
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void archiveErrorFiles(final File source, final File target) throws IOException
public static void archiveErrorFiles(final File source, final File target, final DryOption dry) throws IOException
{
if (source == null)
{
@ -246,14 +250,18 @@ public final class Logar
Stats counter = new Stats();
counter.start();
for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName())
for (File directory : Files.of(source).removeHidden().keepDirectoryType().sortByName())
{
String targetFileName = String.format("%s-error-%s.log.gz", directory.getName(), targetYearMonth.toString());
File targetDirectory = new File(target, directory.getName());
targetDirectory.mkdirs();
File targetFile = new File(targetDirectory, targetFileName);
logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
PrintWriter out = null;
if (dry.isOff())
{
targetDirectory.mkdirs();
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
}
for (File file : Files.of(directory).sortByName())
{
@ -270,14 +278,17 @@ public final class Logar
try
{
Log log = LogUtils.parseErrorLog(line);
Log log = LogParser.parseErrorLog(line);
counter.incSuccessLineCount();
if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
{
if (dry.isOff())
{
out.println(line);
}
}
}
catch (IllegalArgumentException exception)
{
System.err.println("Bad line format [" + line + "]");
@ -326,7 +337,7 @@ public final class Logar
}
else
{
System.out.println("== Check access log for [" + file.getName() + "]");
System.out.println("== Check parse log for [" + file.getName() + "]");
boolean isAccessFile;
if (file.getName().contains("access"))
{
@ -351,11 +362,11 @@ public final class Logar
{
if (isAccessFile)
{
LogUtils.parseAccessLog(line).getDatetime();
LogParser.parseAccessLog(line).getDatetime();
}
else
{
LogUtils.parseErrorLog(line).getDatetime();
LogParser.parseErrorLog(line).getDatetime();
}
}
catch (IllegalArgumentException exception)
@ -392,7 +403,7 @@ public final class Logar
*/
public static void checkLogFiles(final File source)
{
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz", ".1", ".2", ".3").removeHidden().sortByName();
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
for (File file : files)
{
@ -400,6 +411,23 @@ public final class Logar
}
}
/**
* Check sort.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSort(final File source) throws IOException
{
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
for (File file : files)
{
checkSortOfFile(file);
}
}
/**
* Check sort for access files.
*
@ -407,7 +435,7 @@ public final class Logar
* the source
* @throws IOException
*/
public static void checkSort(final File file) throws IOException
public static void checkSortOfFile(final File file) throws IOException
{
if (file == null)
{
@ -441,11 +469,11 @@ public final class Logar
LocalDateTime date;
if (isAccessFile)
{
date = LogUtils.parseAccessLog(line).getDatetime();
date = LogParser.parseAccessLog(line).getDatetime();
}
else
{
date = LogUtils.parseErrorLog(line).getDatetime();
date = LogParser.parseErrorLog(line).getDatetime();
}
if ((currentDate != null) && (date.isBefore(currentDate)))
@ -463,23 +491,6 @@ public final class Logar
}
}
/**
* Check sort.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSorts(final File source) throws IOException
{
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName();
for (File file : files)
{
checkSort(file);
}
}
/**
* Sort.
*
@ -495,189 +506,113 @@ public final class Logar
for (File file : files)
{
System.out.println("== Sort for [" + file.getName() + "]");
LogUtils.sortLogFile(file);
LogFile.sortLogFile(file);
}
}
/**
* Test parsing.
* Check concate.
*
* @param source
* the source
*/
public static void testParsing(final File source)
public static void testConcate(final File source)
{
testParsingAccessFiles(source);
testParsingErrorFiles(source);
Files files = FilesUtils.searchRecursively(source, LOGFILE_PATTERN).sortByName();
for (File file : files)
{
testConcateOnFile(file);
}
}
/**
* Test parsing access files.
* Check concate on file.
*
* @param source
* the source
*/
public static void testParsingAccessFiles(final File source)
public static void testConcateOnFile(final File file)
{
if (source == null)
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
throw new IllegalArgumentException("Null parameter.");
}
else if (!source.isDirectory())
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a directory.");
throw new IllegalArgumentException("Parameter is not a file.");
}
else
{
System.out.println("== Test access log for [" + source.getName() + "]");
Stats counter = new Stats();
for (File directory : source.listFiles())
{
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
{
logger.info("== {}", directory.getName());
for (File file : directory.listFiles())
{
if ((!file.isDirectory()) && (file.getName().contains("access")))
{
// logger.info(file.getName());
System.out.println("== Test concate log for [" + file.getName() + "]");
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
int lineCount = 0;
int badLineCount = 0;
try
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
counter.incLineCount();
lineCount += 1;
Matcher matcher = nginxAccessLogLinePattern.matcher(line);
if (matcher.matches())
{
String value = matcher.group("time");
try
{
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
counter.incSuccessLineCount();
}
catch (DateTimeParseException exception)
Log source;
Log target;
if (isAccessFile)
{
System.err.println("Date format error with [" + line + "]");
counter.incErrorLineCount();
}
source = LogParser.parseAccessLog(line);
target = new Log(source);
target.concateAccessLog();
}
else
{
System.err.println("Not matching line [" + line + "]");
counter.incErrorLineCount();
}
source = LogParser.parseErrorLog(line);
target = new Log(source);
target.concateErrorLog();
}
counter.incSuccessFileCount();
if (!StringUtils.equals(source.getLine(), target.getLine()))
{
System.out.println("Bad concat (1): " + source);
System.out.println("Bad concat (2): " + target);
badLineCount += 1;
}
}
catch (IllegalArgumentException exception)
{
System.out.println("Bad format line: " + line);
badLineCount += 1;
exception.printStackTrace();
}
catch (DateTimeParseException exception)
{
System.out.println("Bad datetime format: " + line);
badLineCount += 1;
}
}
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
counter.incErrorFileCount();
}
finally
{
counter.incFileCount();
}
}
}
}
}
counter.stop();
System.out.println(counter.toString());
}
}
/**
* Test parsing error files.
*
* @param source
* the source
*/
public static void testParsingErrorFiles(final File source)
if (badLineCount > 0)
{
if (source == null)
{
throw new IllegalArgumentException("Null parameter [source]");
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
else if (!source.isDirectory())
{
throw new IllegalArgumentException("Source parameter is not a directory.");
}
else
{
System.out.println("== Test error log for [" + source.getName() + "]");
Stats counter = new Stats();
for (File directory : source.listFiles())
{
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
{
logger.info("== {}", directory.getName());
for (File file : directory.listFiles())
{
if ((!file.isDirectory()) && (file.getName().contains("error")))
{
// logger.info(file.getName());
try
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
counter.incLineCount();
Matcher matcher = nginxErrorLogLinePattern.matcher(line);
if (matcher.matches())
{
String value = matcher.group("time");
try
{
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
counter.incSuccessLineCount();
}
catch (DateTimeParseException exception)
{
System.err.println("Date format problem with [" + line + "]");
counter.incErrorLineCount();
}
}
else
{
System.err.println("Not matching line [" + line + "]");
counter.incErrorLineCount();
}
}
counter.incSuccessFileCount();
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
counter.incErrorFileCount();
}
finally
{
counter.incFileCount();
}
}
}
}
}
System.out.println("=====================================================");
counter.stop();
System.out.println(counter.toString());
}
}
}

28
src/fr/devinsy/logar/app/anonymizer/Anonymizer.java
View file

@ -22,6 +22,8 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.format.DateTimeParseException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -34,7 +36,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogUtils;
import fr.devinsy.logar.app.log.LogParser;
/**
* The Class Anonymizer.
@ -120,7 +122,7 @@ public final class Anonymizer
Log anon;
if (isAccessFile)
{
Log log = LogUtils.parseAccessLog(line);
Log log = LogParser.parseAccessLog(line);
// logger.info("line={}", line);
// logger.info("log =[{}][{}][{}]", log.getIp(),
// log.getUser(), log.getDatetime());
@ -132,7 +134,7 @@ public final class Anonymizer
}
else
{
Log log = LogUtils.parseErrorLog(line);
Log log = LogParser.parseErrorLog(line);
anon = anonymizeError(log);
}
@ -173,16 +175,26 @@ public final class Anonymizer
{
Log result;
String anonIp = this.map.anonymizeIp(log.getIp());
String anonUser = this.map.anonymizeUser(log.getUser());
result = new Log(log);
String line = log.getLine().replace(log.getIp(), anonIp);
result.setIp(this.map.anonymizeIp(log.getIp()));
result.setUser(this.map.anonymizeUser(log.getUser()));
// Anonymize ip.
result.setRequest(result.getRequest().replace(result.getIp(), result.getIp()));
result.setReferer(result.getReferer().replace(result.getIp(), result.getIp()));
// Anonymize user.
if (!log.getUser().equals("-"))
{
line = line.replace(log.getUser(), anonUser);
// URLEncode replaces ' ' with '+' so bad for us.
String userInUrl = URLEncoder.encode(log.getUser(), StandardCharsets.UTF_8).replace("+", "%20");
result.setRequest(result.getRequest().replace(userInUrl, result.getUser()));
result.setReferer(result.getReferer().replace(userInUrl, result.getUser()));
}
result = new Log(line, log.getDatetime(), anonIp, anonUser);
result.concateAccessLog();
//
return result;

181
src/fr/devinsy/logar/app/log/Log.java
View file

@ -33,13 +33,44 @@ public final class Log
// Generic attributes.
private String line;
private LocalDateTime datetime;
private String datetimeValue;
// Specific access log attributes.
private String ip;
private String user;
private String request;
private String status;
private String bodyByteSent;
private String referer;
private String userAgent;
// Specific error log attributes.
// private String message;
private String level;
private String message;
/**
* Instantiates a new log.
*
* @param log
* the log
*/
public Log(final Log log)
{
this.line = log.getLine();
this.datetime = log.getDatetime();
this.datetimeValue = log.getDatetimeValue();
this.ip = log.getIp();
this.user = log.getUser();
this.request = log.getRequest();
this.status = log.getStatus();
this.bodyByteSent = log.getBodyByteSent();
this.referer = log.getReferer();
this.userAgent = log.getUserAgent();
this.level = log.getLevel();
this.message = log.getMessage();
}
/**
* Instantiates a new log.
@ -48,28 +79,39 @@ public final class Log
{
this.line = line;
this.datetime = datetime;
this.datetimeValue = null;
this.ip = null;
this.user = null;
this.request = null;
this.referer = null;
this.userAgent = null;
this.level = null;
this.message = null;
}
/**
* Instantiates a new log.
*
* @param line
* the line
* @param datetime
* the datetime
* @param ip
* the ip
* @param user
* the login
* Concate access log.
*/
public Log(final String line, final LocalDateTime datetime, final String ip, final String user)
public void concateAccessLog()
{
this.line = line;
this.datetime = datetime;
this.ip = ip;
this.user = user;
this.line = String.format("%s - %s [%s] \"%s\" %s %s \"%s\" \"%s\"",
this.ip, this.user, this.datetimeValue, this.request, this.status, this.bodyByteSent, this.referer, this.userAgent);
}
/**
* Concate error log.
*/
public void concateErrorLog()
{
this.line = String.format("%s [%s] %s",
this.datetimeValue, this.level, this.message);
}
public String getBodyByteSent()
{
return this.bodyByteSent;
}
public LocalDateTime getDatetime()
@ -77,21 +119,128 @@ public final class Log
return this.datetime;
}
public String getDatetimeValue()
{
return this.datetimeValue;
}
public String getIp()
{
return this.ip;
}
public String getLevel()
{
return this.level;
}
public String getLine()
{
return this.line;
}
public String getMessage()
{
return this.message;
}
public String getReferer()
{
return this.referer;
}
public String getRequest()
{
return this.request;
}
public String getStatus()
{
return this.status;
}
public String getUser()
{
return this.user;
}
public String getUserAgent()
{
return this.userAgent;
}
/**
* Reduce.
*/
public void reduce()
{
this.datetimeValue = null;
this.request = null;
this.referer = null;
this.userAgent = null;
this.status = null;
this.bodyByteSent = null;
this.level = null;
this.message = null;
}
public void setBodyByteSent(final String bodyByteSent)
{
this.bodyByteSent = bodyByteSent;
}
public void setDatetimeValue(final String datetimeValue)
{
this.datetimeValue = datetimeValue;
}
public void setIp(final String ip)
{
this.ip = ip;
}
public void setLevel(final String level)
{
this.level = level;
}
public void setMessage(final String message)
{
this.message = message;
}
public void setReferer(final String referer)
{
this.referer = referer;
}
public void setRequest(final String request)
{
this.request = request;
}
public void setStatus(final String status)
{
this.status = status;
}
public void setUser(final String user)
{
this.user = user;
}
/**
* Sets the user agent.
*
* @param userAgent
* the new user agent
*/
public void setUserAgent(final String userAgent)
{
this.userAgent = userAgent;
}
/* (non-Javadoc)
* @see java.lang.Object#toString()
*/

256
src/fr/devinsy/logar/app/log/LogFile.java Normal file
View file

@ -0,0 +1,256 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;
import org.april.logar.util.LineIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.cmdexec.CmdExecException;
import fr.devinsy.cmdexec.CmdExecUtils;
/**
* The Class LogFile.
*/
public final class LogFile
{
private static Logger logger = LoggerFactory.getLogger(LogFile.class);
/**
* Instantiates a new log file.
*/
private LogFile()
{
}
/**
* Load access log.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadAccessLog(final File file) throws IOException
{
Logs result;
result = loadAccessLog(file, LogMode.NORMAL);
return result;
}
/**
* Load access log.
*
* @param file
* the file
* @param mode
* the mode
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadAccessLog(final File file, final LogMode mode) throws IOException
{
Logs result;
result = new Logs();
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
Log log = LogParser.parseAccessLog(line);
if (mode == LogMode.REDUCED)
{
log.reduce();
}
result.add(log);
}
//
return result;
}
/**
* Load error log.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadErrorLog(final File file) throws IOException
{
Logs result;
result = new Logs();
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
Log log = LogParser.parseErrorLog(line);
result.add(log);
}
//
return result;
}
/**
* Load log file.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadLogFile(final File file) throws IOException
{
Logs result;
result = loadLogFile(file, LogMode.NORMAL);
//
return result;
}
/**
* Load log file.
*
* @param file
* the file
* @param mode
* the mode
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadLogFile(final File file, final LogMode mode) throws IOException
{
Logs result;
if (file == null)
{
throw new IllegalArgumentException("Null parameter.");
}
else
{
if (file.getName().contains("access"))
{
result = loadAccessLog(file, mode);
}
else if (file.getName().contains("error"))
{
result = loadErrorLog(file);
}
else
{
throw new IllegalArgumentException("Bad named file (missing access or error).");
}
}
//
return result;
}
/**
* Save access log.
*
* @param source
* the source
* @throws IOException
* @throws FileNotFoundException
*/
public static void saveLogs(final File target, final Logs logs) throws FileNotFoundException, IOException
{
PrintWriter out = null;
try
{
if (target.getName().endsWith(".gz"))
{
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
}
else
{
out = new PrintWriter(new FileOutputStream(target));
}
for (Log log : logs)
{
out.println(log.getLine());
}
}
finally
{
IOUtils.closeQuietly(out);
}
}
/**
* Sort access log.
*
* @param target
* the target
* @throws IOException
*/
public static void sortLogFile(final File file) throws IOException
{
File workFile = new File(file.getParent(), file.getName() + ".tmp");
Logs logs = loadLogFile(file, LogMode.REDUCED);
logs.sortByDatetime();
saveLogs(workFile, logs);
File backup = new File(file.getParentFile(), file.getName() + ".bak");
if (file.renameTo(backup))
{
if (!workFile.renameTo(file))
{
backup.renameTo(file);
}
}
// Check.
try
{
String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \"");
System.out.print(out);
out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum \"");
System.out.println(out);
}
catch (CmdExecException exception)
{
exception.printStackTrace();
}
}
}

28
src/fr/devinsy/logar/app/log/LogMode.java Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log;
/**
* The Enum LogMode.
*/
public enum LogMode
{
NORMAL,
REDUCED;
}

153
src/fr/devinsy/logar/app/log/LogParser.java Normal file
View file

@ -0,0 +1,153 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class LogParser.
*/
public final class LogParser
{
private static Logger logger = LoggerFactory.getLogger(LogParser.class);
public static Pattern NGINX_ACCESSLOG_LINE_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<datetime>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?<datetime>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
/**
* Instantiates a new log parser.
*/
private LogParser()
{
}
/**
* From access log.
*
* @param line
* the line
* @return the log
*
* log_format combined '$remote_addr - $remote_user [$time_local] '
* '"$request" $status $body_bytes_sent ' '"$http_referer"
* "$http_user_agent"';
*/
public static Log parseAccessLog(final String line)
{
Log result;
try
{
Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String dateTimeValue = matcher.group("datetime");
LocalDateTime dateTime = LocalDateTime.parse(dateTimeValue, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
result = new Log(line, dateTime);
result.setDatetimeValue(matcher.group("datetime"));
result.setIp(matcher.group("remoteAddress"));
result.setUser(matcher.group("remoteUser"));
result.setRequest(matcher.group("request"));
result.setStatus(matcher.group("status"));
result.setBodyByteSent(matcher.group("bodyBytesSent"));
result.setReferer(matcher.group("referer"));
result.setUserAgent(matcher.group("userAgent"));
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line);
}
//
return result;
}
/**
* Parses the access log light.
*
* @param line
* the line
* @return the log
*/
public static Log parseAccessLogLight(final String line)
{
Log result;
result = parseAccessLog(line);
result.reduce();
//
return result;
}
/**
* From error log.
*
* @param line
* the line
* @return the log
*/
public static Log parseErrorLog(final String line)
{
Log result;
try
{
Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String value = matcher.group("datetime");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
result = new Log(line, date);
result.setDatetimeValue(matcher.group("datetime"));
result.setLevel(matcher.group("level"));
result.setMessage(matcher.group("message"));
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line);
}
//
return result;
}
}

262
src/fr/devinsy/logar/app/log/LogUtils.java
View file

@ -18,278 +18,20 @@
*/
package fr.devinsy.logar.app.log;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
import org.apache.commons.io.IOUtils;
import org.april.logar.util.LineIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.cmdexec.CmdExecException;
import fr.devinsy.cmdexec.CmdExecUtils;
/**
* The Class NginxAccessLogParser.
* The Class LogUtils.
*/
public final class LogUtils
{
private static Logger logger = LoggerFactory.getLogger(LogUtils.class);
public static Pattern nginxAccessLogLinePatternFull = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>\\S+) \\[(?<time>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
public static Pattern NGINX_ACCESSLOG_LINE_PATTERN = Pattern.compile("^(?<remoteAddress>[a-fA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<time>[^\\]]+)\\] .*$");
public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?<time>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
/**
* Instantiates a new nginx access log parser.
* Instantiates a new log utils.
*/
private LogUtils()
{
}
/**
* Load.
*
* @param file
* the file
* @return the logs
* @throws IOException
*/
public static Logs loadAccessLog(final File file) throws IOException
{
Logs result;
result = new Logs();
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
Log log = parseAccessLog(line);
result.add(log);
}
//
return result;
}
/**
* Load error log.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadErrorLog(final File file) throws IOException
{
Logs result;
result = new Logs();
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
Log log = parseErrorLog(line);
result.add(log);
}
//
return result;
}
/**
* Load log file.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadLogFile(final File file) throws IOException
{
Logs result;
if (file == null)
{
throw new IllegalArgumentException("Null parameter.");
}
else
{
if (file.getName().contains("access"))
{
result = loadAccessLog(file);
}
else if (file.getName().contains("error"))
{
result = loadErrorLog(file);
}
else
{
throw new IllegalArgumentException("Bad named file (missing access or error).");
}
}
//
return result;
}
/**
* From access log.
*
* @param line
* the line
* @return the log
*/
public static Log parseAccessLog(final String line)
{
Log result;
try
{
Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String value = matcher.group("time");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
String ip = matcher.group("remoteAddress").trim();
String user = matcher.group("remoteUser").trim();
result = new Log(line, date, ip, user);
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line);
}
//
return result;
}
/**
* From error log.
*
* @param line
* the line
* @return the log
*/
public static Log parseErrorLog(final String line)
{
Log result;
try
{
Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String value = matcher.group("time");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
result = new Log(line, date);
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line);
}
//
return result;
}
/**
* Save access log.
*
* @param source
* the source
* @throws IOException
* @throws FileNotFoundException
*/
public static void saveLogs(final File target, final Logs logs) throws FileNotFoundException, IOException
{
PrintWriter out = null;
try
{
if (target.getName().endsWith(".gz"))
{
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
}
else
{
out = new PrintWriter(new FileOutputStream(target));
}
for (Log log : logs)
{
out.println(log.getLine());
}
}
finally
{
IOUtils.closeQuietly(out);
}
}
/**
* Sort access log.
*
* @param target
* the target
* @throws IOException
*/
public static void sortLogFile(final File file) throws IOException
{
File workFile = new File(file.getParent(), file.getName() + ".tmp");
Logs logs = loadLogFile(file);
logs.sortByDatetime();
saveLogs(workFile, logs);
File backup = new File(file.getParentFile(), file.getName() + ".bak");
if (file.renameTo(backup))
{
if (!workFile.renameTo(file))
{
backup.renameTo(file);
}
}
// Check.
try
{
String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \"");
System.out.print(out);
out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum \"");
System.out.println(out);
}
catch (CmdExecException exception)
{
exception.printStackTrace();
}
}
}

19
src/org/april/logar/cli/LogarCLI.java
View file

@ -25,6 +25,7 @@ import org.april.logar.util.BuildInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.DryOption;
import fr.devinsy.logar.app.Logar;
import fr.devinsy.strings.StringList;
@ -56,10 +57,11 @@ public final class LogarCLI
message.appendln(" logar [ -v | -version | --version ]");
message.appendln(" logar anonymize fileordirectory [maptable] anonymize ip and login");
message.appendln(" logar archive source target archive previous month");
message.appendln(" logar check fileordirectory census bad format line in log files");
message.appendln(" logar check fileordirectory check line format in log files");
message.appendln(" logar checksort fileordirectory check sort of an access log file");
message.appendln(" logar sort fileordirectory sort log files by datetime");
message.appendln(" logar testarchive source test archive");
message.appendln(" logar testconcate fileordirectory test concate of log line");
logger.info(message.toString());
}
@ -201,7 +203,7 @@ public final class LogarCLI
File source = new File(args[1]);
File target = new File(args[2]);
Logar.archive(source, target);
Logar.archive(source, target, DryOption.OFF);
}
else if (isMatching(args, "check", "\\s*\\S+\\s*"))
{
@ -213,7 +215,7 @@ public final class LogarCLI
{
File source = new File(args[1]);
Logar.checkSorts(source);
Logar.checkSort(source);
}
else if (isMatching(args, "sort", "\\s*\\S+\\s*"))
{
@ -221,11 +223,18 @@ public final class LogarCLI
Logar.sort(source);
}
else if (isMatching(args, "testarchive", "\\s*\\S+\\s*"))
else if (isMatching(args, "testarchive", "\\s*\\S+\\s*", "\\s*\\S+\\s*"))
{
File source = new File(args[1]);
File target = new File(args[2]);
Logar.archive(source, target, DryOption.ON);
}
else if (isMatching(args, "testconcate", "\\s*\\S+\\s*"))
{
File source = new File(args[1]);
Logar.testParsing(source);
Logar.testConcate(source);
}
else
{

58
src/org/april/logar/util/Files.java
View file

@ -21,6 +21,7 @@ package org.april.logar.util;
import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.regex.Pattern;
/**
* The Class Files.
@ -48,12 +49,42 @@ public class Files extends ArrayList<File>
super(initialCapacity);
}
/**
* Keep.
*
* @param regex
* the regex
* @return the files
*/
public Files keep(final String regex)
{
Files result;
Pattern pattern = Pattern.compile(regex);
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!pattern.matcher(file.getName()).matches())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Keep directories.
*
* @return the files
*/
public Files keepDirectoriesOnly()
public Files keepDirectoryType()
{
Files result;
@ -73,6 +104,31 @@ public class Files extends ArrayList<File>
return result;
}
/**
* Keep file type.
*
* @return the files
*/
public Files keepFileType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isFile())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Removes the containing.
*

31
src/org/april/logar/util/FilesUtils.java
View file

@ -19,6 +19,7 @@
package org.april.logar.util;
import java.io.File;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
@ -100,4 +101,34 @@ public class FilesUtils
//
return result;
}
/**
* Search recursively.
*
* @param source
* the source
* @param regex
* the regex
* @return the files
*/
public static Files searchRecursively(final File source, final String regex)
{
Files result;
result = new Files();
Pattern pattern = Pattern.compile(regex);
Files full = listRecursively(source);
for (File file : full)
{
if (pattern.matcher(file.getName()).matches())
{
result.add(file);
}
}
//
return result;
}
}