statool/logar
statool/logar
1
0
Fork 0
Code Issues Pull requests Releases Activity

Refactored code.

Browse source
This commit is contained in:
Christian P. MOMON 2021-04-22 20:21:22 +02:00
parent 541e0d9254
commit c3a6fc9841
5 changed files with 301 additions and 424 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

521
src/fr/devinsy/logar/app/Logar.java
View file

@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.anonymizer.Anonymizer;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogUtils;
/**
@ -100,14 +101,7 @@ public final class Logar
logger.info("file count={}", files.size());
for (File file : files)
{
if (file.getName().contains("access"))
{
anonymizer.anonymize(file);
}
else if (file.getName().contains("error"))
{
//
}
anonymizer.anonymize(file);
}
if (mapFile != null)
@ -159,74 +153,61 @@ public final class Logar
YearMonth targetYearMonth = YearMonth.now().minusMonths(1);
Stats counter = new Stats();
for (File directory : Files.of(source).sortByName())
for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName())
{
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString());
File targetDirectory = new File(target, directory.getName());
targetDirectory.mkdirs();
File targetFile = new File(targetDirectory, targetFileName);
logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
for (File file : Files.of(directory).sortByName())
{
String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString());
File targetDirectory = new File(target, directory.getName());
targetDirectory.mkdirs();
File targetFile = new File(targetDirectory, targetFileName);
logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
for (File file : Files.of(directory).sortByName())
if ((!file.isDirectory()) && (file.getName().contains("access")))
{
if ((!file.isDirectory()) && (file.getName().contains("access")))
logger.info(file.getName());
try
{
logger.info(file.getName());
try
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
String line = iterator.next();
counter.incLineCount();
try
{
String line = iterator.next();
counter.incLineCount();
Log log = LogUtils.parseAccessLog(line);
counter.incSuccessLineCount();
Matcher matcher = nginxAccessLogLinePattern.matcher(line);
if (matcher.matches())
if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
{
String value = matcher.group("time");
try
{
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
counter.incSuccessLineCount();
if (YearMonth.from(date).equals(targetYearMonth))
{
out.println(line);
}
}
catch (DateTimeParseException exception)
{
System.err.println("Date format problem with [" + line + "]");
counter.incErrorLineCount();
}
}
else
{
System.err.println("Not matching line [" + line + "]");
counter.incErrorLineCount();
out.println(line);
}
}
catch (IllegalArgumentException exception)
{
System.err.println("Bad line format [" + line + "]");
counter.incErrorLineCount();
}
}
counter.incSuccessFileCount();
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
counter.incErrorFileCount();
}
finally
{
counter.incFileCount();
}
counter.incSuccessFileCount();
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
counter.incErrorFileCount();
}
finally
{
counter.incFileCount();
}
}
IOUtils.closeQuietly(out);
}
IOUtils.closeQuietly(out);
}
System.out.println("=====================================================");
@ -265,74 +246,60 @@ public final class Logar
Stats counter = new Stats();
counter.start();
for (File directory : Files.of(source).sortByName())
for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName())
{
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
String targetFileName = String.format("%s-error-%s.log.gz", directory.getName(), targetYearMonth.toString());
File targetDirectory = new File(target, directory.getName());
targetDirectory.mkdirs();
File targetFile = new File(targetDirectory, targetFileName);
logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
for (File file : Files.of(directory).sortByName())
{
String targetFileName = String.format("%s-error-%s.log.gz", directory.getName(), targetYearMonth.toString());
File targetDirectory = new File(target, directory.getName());
targetDirectory.mkdirs();
File targetFile = new File(targetDirectory, targetFileName);
logger.info("== {} -> {}", directory.getName(), targetFile.getAbsoluteFile());
PrintWriter out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(targetFile)));
for (File file : Files.of(directory).sortByName())
if ((!file.isDirectory()) && (file.getName().contains("error")))
{
if ((!file.isDirectory()) && (file.getName().contains("error")))
// logger.info(file.getName());
try
{
// logger.info(file.getName());
try
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
String line = iterator.next();
counter.incLineCount();
try
{
String line = iterator.next();
counter.incLineCount();
Log log = LogUtils.parseErrorLog(line);
counter.incSuccessLineCount();
Matcher matcher = nginxErrorLogLinePattern.matcher(line);
if (matcher.matches())
if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
{
String value = matcher.group("time");
try
{
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
counter.incSuccessLineCount();
if (YearMonth.from(date).equals(targetYearMonth))
{
out.println(line);
}
}
catch (DateTimeParseException exception)
{
System.err.println("Date format problem with [" + line + "]");
counter.incErrorLineCount();
}
}
else
{
System.err.println("Not matching line [" + line + "]");
counter.incErrorLineCount();
out.println(line);
}
}
counter.incSuccessLineCount();
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
counter.incErrorFileCount();
}
finally
{
counter.incFileCount();
catch (IllegalArgumentException exception)
{
System.err.println("Bad line format [" + line + "]");
counter.incErrorLineCount();
}
}
counter.incSuccessFileCount();
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
counter.incErrorFileCount();
}
finally
{
counter.incFileCount();
}
}
IOUtils.closeQuietly(out);
}
IOUtils.closeQuietly(out);
}
System.out.println("=====================================================");
@ -341,51 +308,13 @@ public final class Logar
}
}
/**
* Check.
*
* @param source
* the source
*/
public static void check(final File source)
{
if (source != null)
{
if (source.isFile())
{
if (source.getName().contains("access"))
{
checkAccessFiles(source);
}
else if (source.getName().contains("error"))
{
checkErrorFiles(source);
}
}
else
{
for (File file : Files.of(source).removeHidden().sortByName())
{
if (file.isDirectory())
{
check(file);
}
else if (StringUtils.endsWithAny(file.getName(), ".log", ".gz", ".1", ".2"))
{
check(file);
}
}
}
}
}
/**
* Check access files.
*
* @param file
* the source
*/
public static void checkAccessFiles(final File file)
public static void checkLogFile(final File file)
{
if (file == null)
{
@ -398,6 +327,15 @@ public final class Logar
else
{
System.out.println("== Check access log for [" + file.getName() + "]");
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
int lineCount = 0;
int badLineCount = 0;
@ -411,7 +349,14 @@ public final class Logar
try
{
LogUtils.parseAccessLog(line).getDatetime();
if (isAccessFile)
{
LogUtils.parseAccessLog(line).getDatetime();
}
else
{
LogUtils.parseErrorLog(line).getDatetime();
}
}
catch (IllegalArgumentException exception)
{
@ -440,55 +385,75 @@ public final class Logar
}
/**
* Check error files.
* Check.
*
* @param file
* the file
* @param source
* the source
*/
public static void checkErrorFiles(final File file)
public static void checkLogFiles(final File source)
{
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz", ".1", ".2", ".3").removeHidden().sortByName();
for (File file : files)
{
checkLogFile(file);
}
}
/**
* Check sort for access files.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSort(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter.");
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Parameter is not a file.");
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Check error log for [" + file.getName() + "]");
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
System.out.println("== Check sort for [" + file.getName() + "]");
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
try
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
String line = iterator.next();
lineCount += 1;
LocalDateTime date;
if (isAccessFile)
{
String line = iterator.next();
lineCount += 1;
try
{
LogUtils.parseErrorLog(line).getDatetime();
}
catch (IllegalArgumentException exception)
{
System.out.println("Bad format line: " + line);
badLineCount += 1;
}
catch (DateTimeParseException exception)
{
System.out.println("Bad datetime format: " + line);
badLineCount += 1;
}
date = LogUtils.parseAccessLog(line).getDatetime();
}
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
else
{
date = LogUtils.parseErrorLog(line).getDatetime();
}
if ((currentDate != null) && (date.isBefore(currentDate)))
{
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
badLineCount += 1;
}
currentDate = date;
}
if (badLineCount > 0)
@ -505,116 +470,13 @@ public final class Logar
* the source
* @throws IOException
*/
public static void checkSort(final File source) throws IOException
public static void checkSorts(final File source) throws IOException
{
if (source != null)
{
if (source.isFile())
{
if (source.getName().contains("access"))
{
checkSortForAccessFiles(source);
}
else if (source.getName().contains("error"))
{
checkSortForErrorFiles(source);
}
}
else
{
for (File file : Files.of(source).removeHidden().sortByName())
{
checkSort(file);
}
}
}
}
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName();
/**
* Check sort for access files.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSortForAccessFiles(final File file) throws IOException
{
if (file == null)
for (File file : files)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Check sort for [" + file.getName() + "]");
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
lineCount += 1;
LocalDateTime date = LogUtils.parseAccessLog(line).getDatetime();
if ((currentDate != null) && (date.isBefore(currentDate)))
{
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
badLineCount += 1;
}
currentDate = date;
}
if (badLineCount > 0)
{
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
}
}
/**
* Check sort for error files.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSortForErrorFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Check sort for [" + file.getName() + "]");
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
lineCount += 1;
LocalDateTime date = LogUtils.parseErrorLog(line).getDatetime();
if ((currentDate != null) && (date.isBefore(currentDate)))
{
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
badLineCount += 1;
}
currentDate = date;
}
if (badLineCount > 0)
{
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
checkSort(file);
}
}
@ -628,75 +490,12 @@ public final class Logar
*/
public static void sort(final File source) throws IOException
{
if (source != null)
{
if (source.isFile())
{
if (source.getName().contains("access"))
{
sortAccessFiles(source);
}
else if (source.getName().contains("error"))
{
sortErrorFiles(source);
}
}
else
{
for (File file : Files.of(source).removeHidden().sortByName())
{
sort(file);
}
}
}
}
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName();
/**
* Sort access files.
*
* @param file
* the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void sortAccessFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
for (File file : files)
{
System.out.println("== Sort for [" + file.getName() + "]");
LogUtils.sortAccessLog(file);
}
}
/**
* Sort error files.
*
* @param file
* the file
* @throws IOException
*/
public static void sortErrorFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Sort for [" + file.getName() + "]");
LogUtils.sortErrorLog(file);
LogUtils.sortLogFile(file);
}
}

6
src/fr/devinsy/logar/app/log/Log.java
View file

@ -30,11 +30,17 @@ public final class Log
{
private static Logger logger = LoggerFactory.getLogger(Log.class);
// Generic attributes.
private String line;
private LocalDateTime datetime;
// Specific access log attributes.
private String ip;
private String user;
// Specific error log attributes.
// private String message;
/**
* Instantiates a new log.
*/

142
src/fr/devinsy/logar/app/log/LogUtils.java
View file

@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -111,6 +112,43 @@ public final class LogUtils
return result;
}
/**
* Load log file.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadLogFile(final File file) throws IOException
{
Logs result;
if (file == null)
{
throw new IllegalArgumentException("Null parameter.");
}
else
{
if (file.getName().contains("access"))
{
result = loadAccessLog(file);
}
else if (file.getName().contains("error"))
{
result = loadErrorLog(file);
}
else
{
throw new IllegalArgumentException("Bad named file (missing access or error).");
}
}
//
return result;
}
/**
* From access log.
*
@ -122,20 +160,27 @@ public final class LogUtils
{
Log result;
Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
try
{
String value = matcher.group("time");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String value = matcher.group("time");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
String ip = matcher.group("remoteAddress").trim();
String user = matcher.group("remoteUser").trim();
String ip = matcher.group("remoteAddress").trim();
String user = matcher.group("remoteUser").trim();
result = new Log(line, date, ip, user);
result = new Log(line, date, ip, user);
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
else
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format: " + line);
throw new IllegalArgumentException("Bad line format (date): " + line);
}
//
@ -153,17 +198,24 @@ public final class LogUtils
{
Log result;
Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
try
{
String value = matcher.group("time");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String value = matcher.group("time");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
result = new Log(line, date);
result = new Log(line, date);
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
else
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format: " + line);
throw new IllegalArgumentException("Bad line format (date): " + line);
}
//
@ -183,7 +235,14 @@ public final class LogUtils
PrintWriter out = null;
try
{
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
if (target.getName().endsWith(".gz"))
{
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
}
else
{
out = new PrintWriter(new FileOutputStream(target));
}
for (Log log : logs)
{
@ -203,61 +262,24 @@ public final class LogUtils
* the target
* @throws IOException
*/
public static void sortAccessLog(final File file) throws IOException
public static void sortLogFile(final File file) throws IOException
{
File work = new File(file.getParent(), file.getName() + ".tmp");
File workFile = new File(file.getParent(), file.getName() + ".tmp");
Logs logs = loadAccessLog(file);
Logs logs = loadLogFile(file);
logs.sortByDatetime();
saveLogs(work, logs);
saveLogs(workFile, logs);
File backup = new File(file.getParentFile(), file.getName() + ".bak");
if (file.renameTo(backup))
{
if (!work.renameTo(file))
{
backup.renameTo(file);
}
}
try
{
String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \"");
System.out.print(out);
out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum \"");
System.out.println(out);
}
catch (CmdExecException exception)
{
exception.printStackTrace();
}
}
/**
* Sort error log.
*
* @param file
* the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void sortErrorLog(final File file) throws IOException
{
File work = new File(file.getParent(), file.getName() + ".tmp");
Logs logs = loadErrorLog(file);
logs.sortByDatetime();
saveLogs(work, logs);
File backup = new File(file.getParentFile(), file.getName() + ".bak");
if (file.renameTo(backup))
{
if (!work.renameTo(file))
if (!workFile.renameTo(file))
{
backup.renameTo(file);
}
}
// Check.
try
{
String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \"");

6
src/org/april/logar/cli/LogarCLI.java
View file

@ -56,8 +56,8 @@ public final class LogarCLI
message.appendln(" logar [ -v | -version | --version ]");
message.appendln(" logar anonymize fileordirectory [maptable] anonymize ip and login");
message.appendln(" logar archive source target archive previous month");
message.appendln(" logar checksort fileordirectory check sort of an access log file");
message.appendln(" logar check fileordirectory census bad format line in log files");
message.appendln(" logar checksort fileordirectory check sort of an access log file");
message.appendln(" logar sort fileordirectory sort log files by datetime");
message.appendln(" logar testarchive source test archive");
@ -207,13 +207,13 @@ public final class LogarCLI
{
File source = new File(args[1]);
Logar.check(source);
Logar.checkLogFiles(source);
}
else if (isMatching(args, "checksort", "\\s*\\S+\\s*"))
{
File source = new File(args[1]);
Logar.checkSort(source);
Logar.checkSorts(source);
}
else if (isMatching(args, "sort", "\\s*\\S+\\s*"))
{

50
src/org/april/logar/util/Files.java
View file

@ -48,6 +48,31 @@ public class Files extends ArrayList<File>
super(initialCapacity);
}
/**
* Keep directories.
*
* @return the files
*/
public Files keepDirectoriesOnly()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isDirectory())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Removes the containing.
*
@ -75,6 +100,31 @@ public class Files extends ArrayList<File>
return result;
}
/**
* Removes the file type.
*
* @return the files
*/
public Files removeFileType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (file.isFile())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Removes the hidden.
*