statool/logar
statool/logar
1
0
Fork 0
Code Issues Pull requests Releases Activity

Refactored code.

Browse source
This commit is contained in:
Christian P. MOMON 2021-04-22 20:21:22 +02:00
parent 541e0d9254
commit c3a6fc9841
5 changed files with 301 additions and 424 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

365
src/fr/devinsy/logar/app/Logar.java
View file

@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.anonymizer.Anonymizer;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogUtils;
/**
@ -99,16 +100,9 @@ public final class Logar
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeContaining("-anon.log");
logger.info("file count={}", files.size());
for (File file : files)
{
if (file.getName().contains("access"))
{
anonymizer.anonymize(file);
}
else if (file.getName().contains("error"))
{
//
}
}
if (mapFile != null)
{
@ -159,9 +153,7 @@ public final class Logar
YearMonth targetYearMonth = YearMonth.now().minusMonths(1);
Stats counter = new Stats();
for (File directory : Files.of(source).sortByName())
{
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName())
{
String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString());
File targetDirectory = new File(target, directory.getName());
@ -183,29 +175,19 @@ public final class Logar
String line = iterator.next();
counter.incLineCount();
Matcher matcher = nginxAccessLogLinePattern.matcher(line);
if (matcher.matches())
{
String value = matcher.group("time");
try
{
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH));
Log log = LogUtils.parseAccessLog(line);
counter.incSuccessLineCount();
if (YearMonth.from(date).equals(targetYearMonth))
if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
{
out.println(line);
}
}
catch (DateTimeParseException exception)
catch (IllegalArgumentException exception)
{
System.err.println("Date format problem with [" + line + "]");
counter.incErrorLineCount();
}
}
else
{
System.err.println("Not matching line [" + line + "]");
System.err.println("Bad line format [" + line + "]");
counter.incErrorLineCount();
}
}
@ -227,7 +209,6 @@ public final class Logar
IOUtils.closeQuietly(out);
}
}
System.out.println("=====================================================");
counter.stop();
@ -265,9 +246,7 @@ public final class Logar
Stats counter = new Stats();
counter.start();
for (File directory : Files.of(source).sortByName())
{
if ((directory.isDirectory()) && (!StringUtils.equalsAny(directory.getName(), ".", "..")))
for (File directory : Files.of(source).removeHidden().keepDirectoriesOnly().sortByName())
{
String targetFileName = String.format("%s-error-%s.log.gz", directory.getName(), targetYearMonth.toString());
File targetDirectory = new File(target, directory.getName());
@ -289,34 +268,23 @@ public final class Logar
String line = iterator.next();
counter.incLineCount();
Matcher matcher = nginxErrorLogLinePattern.matcher(line);
if (matcher.matches())
{
String value = matcher.group("time");
try
{
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
Log log = LogUtils.parseErrorLog(line);
counter.incSuccessLineCount();
if (YearMonth.from(date).equals(targetYearMonth))
if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
{
out.println(line);
}
}
catch (DateTimeParseException exception)
catch (IllegalArgumentException exception)
{
System.err.println("Date format problem with [" + line + "]");
System.err.println("Bad line format [" + line + "]");
counter.incErrorLineCount();
}
}
else
{
System.err.println("Not matching line [" + line + "]");
counter.incErrorLineCount();
}
}
counter.incSuccessLineCount();
counter.incSuccessFileCount();
}
catch (IOException exception)
{
@ -333,7 +301,6 @@ public final class Logar
IOUtils.closeQuietly(out);
}
}
System.out.println("=====================================================");
counter.stop();
@ -341,51 +308,13 @@ public final class Logar
}
}
/**
* Check.
*
* @param source
* the source
*/
public static void check(final File source)
{
if (source != null)
{
if (source.isFile())
{
if (source.getName().contains("access"))
{
checkAccessFiles(source);
}
else if (source.getName().contains("error"))
{
checkErrorFiles(source);
}
}
else
{
for (File file : Files.of(source).removeHidden().sortByName())
{
if (file.isDirectory())
{
check(file);
}
else if (StringUtils.endsWithAny(file.getName(), ".log", ".gz", ".1", ".2"))
{
check(file);
}
}
}
}
}
/**
* Check access files.
*
* @param file
* the source
*/
public static void checkAccessFiles(final File file)
public static void checkLogFile(final File file)
{
if (file == null)
{
@ -398,6 +327,15 @@ public final class Logar
else
{
System.out.println("== Check access log for [" + file.getName() + "]");
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
int lineCount = 0;
int badLineCount = 0;
@ -410,9 +348,16 @@ public final class Logar
lineCount += 1;
try
{
if (isAccessFile)
{
LogUtils.parseAccessLog(line).getDatetime();
}
else
{
LogUtils.parseErrorLog(line).getDatetime();
}
}
catch (IllegalArgumentException exception)
{
System.out.println("Bad format line: " + line);
@ -440,55 +385,75 @@ public final class Logar
}
/**
* Check error files.
* Check.
*
* @param file
* the file
* @param source
* the source
*/
public static void checkErrorFiles(final File file)
public static void checkLogFiles(final File source)
{
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz", ".1", ".2", ".3").removeHidden().sortByName();
for (File file : files)
{
checkLogFile(file);
}
}
/**
* Check sort for access files.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSort(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter.");
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Parameter is not a file.");
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Check error log for [" + file.getName() + "]");
boolean isAccessFile;
if (file.getName().contains("access"))
{
isAccessFile = true;
}
else
{
isAccessFile = false;
}
System.out.println("== Check sort for [" + file.getName() + "]");
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
try
{
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
lineCount += 1;
LocalDateTime date;
if (isAccessFile)
{
date = LogUtils.parseAccessLog(line).getDatetime();
}
else
{
date = LogUtils.parseErrorLog(line).getDatetime();
}
try
if ((currentDate != null) && (date.isBefore(currentDate)))
{
LogUtils.parseErrorLog(line).getDatetime();
}
catch (IllegalArgumentException exception)
{
System.out.println("Bad format line: " + line);
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
badLineCount += 1;
}
catch (DateTimeParseException exception)
{
System.out.println("Bad datetime format: " + line);
badLineCount += 1;
}
}
}
catch (IOException exception)
{
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
exception.printStackTrace();
currentDate = date;
}
if (badLineCount > 0)
@ -505,118 +470,15 @@ public final class Logar
* the source
* @throws IOException
*/
public static void checkSort(final File source) throws IOException
public static void checkSorts(final File source) throws IOException
{
if (source != null)
{
if (source.isFile())
{
if (source.getName().contains("access"))
{
checkSortForAccessFiles(source);
}
else if (source.getName().contains("error"))
{
checkSortForErrorFiles(source);
}
}
else
{
for (File file : Files.of(source).removeHidden().sortByName())
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName();
for (File file : files)
{
checkSort(file);
}
}
}
}
/**
* Check sort for access files.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSortForAccessFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Check sort for [" + file.getName() + "]");
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
lineCount += 1;
LocalDateTime date = LogUtils.parseAccessLog(line).getDatetime();
if ((currentDate != null) && (date.isBefore(currentDate)))
{
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
badLineCount += 1;
}
currentDate = date;
}
if (badLineCount > 0)
{
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
}
}
/**
* Check sort for error files.
*
* @param source
* the source
* @throws IOException
*/
public static void checkSortForErrorFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Check sort for [" + file.getName() + "]");
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
lineCount += 1;
LocalDateTime date = LogUtils.parseErrorLog(line).getDatetime();
if ((currentDate != null) && (date.isBefore(currentDate)))
{
System.out.println(String.format("break detected: %d %s", lineCount, currentDate.toString()));
badLineCount += 1;
}
currentDate = date;
}
if (badLineCount > 0)
{
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
}
}
}
/**
* Sort.
@ -628,75 +490,12 @@ public final class Logar
*/
public static void sort(final File source) throws IOException
{
if (source != null)
{
if (source.isFile())
{
if (source.getName().contains("access"))
{
sortAccessFiles(source);
}
else if (source.getName().contains("error"))
{
sortErrorFiles(source);
}
}
else
{
for (File file : Files.of(source).removeHidden().sortByName())
{
sort(file);
}
}
}
}
Files files = FilesUtils.searchFileRecursively(source, ".log", ".log.gz").removeHidden().sortByName();
/**
* Sort access files.
*
* @param file
* the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void sortAccessFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
for (File file : files)
{
System.out.println("== Sort for [" + file.getName() + "]");
LogUtils.sortAccessLog(file);
}
}
/**
* Sort error files.
*
* @param file
* the file
* @throws IOException
*/
public static void sortErrorFiles(final File file) throws IOException
{
if (file == null)
{
throw new IllegalArgumentException("Null parameter [source]");
}
else if (!file.isFile())
{
throw new IllegalArgumentException("Source parameter is not a file.");
}
else
{
System.out.println("== Sort for [" + file.getName() + "]");
LogUtils.sortErrorLog(file);
LogUtils.sortLogFile(file);
}
}

6
src/fr/devinsy/logar/app/log/Log.java
View file

@ -30,11 +30,17 @@ public final class Log
{
private static Logger logger = LoggerFactory.getLogger(Log.class);
// Generic attributes.
private String line;
private LocalDateTime datetime;
// Specific access log attributes.
private String ip;
private String user;
// Specific error log attributes.
// private String message;
/**
* Instantiates a new log.
*/

108
src/fr/devinsy/logar/app/log/LogUtils.java
View file

@ -25,6 +25,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -111,6 +112,43 @@ public final class LogUtils
return result;
}
/**
* Load log file.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadLogFile(final File file) throws IOException
{
Logs result;
if (file == null)
{
throw new IllegalArgumentException("Null parameter.");
}
else
{
if (file.getName().contains("access"))
{
result = loadAccessLog(file);
}
else if (file.getName().contains("error"))
{
result = loadErrorLog(file);
}
else
{
throw new IllegalArgumentException("Bad named file (missing access or error).");
}
}
//
return result;
}
/**
* From access log.
*
@ -122,6 +160,8 @@ public final class LogUtils
{
Log result;
try
{
Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
@ -137,6 +177,11 @@ public final class LogUtils
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line);
}
//
return result;
@ -153,6 +198,8 @@ public final class LogUtils
{
Log result;
try
{
Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
@ -165,6 +212,11 @@ public final class LogUtils
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line);
}
//
return result;
@ -182,8 +234,15 @@ public final class LogUtils
{
PrintWriter out = null;
try
{
if (target.getName().endsWith(".gz"))
{
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
}
else
{
out = new PrintWriter(new FileOutputStream(target));
}
for (Log log : logs)
{
@ -203,61 +262,24 @@ public final class LogUtils
* the target
* @throws IOException
*/
public static void sortAccessLog(final File file) throws IOException
public static void sortLogFile(final File file) throws IOException
{
File work = new File(file.getParent(), file.getName() + ".tmp");
File workFile = new File(file.getParent(), file.getName() + ".tmp");
Logs logs = loadAccessLog(file);
Logs logs = loadLogFile(file);
logs.sortByDatetime();
saveLogs(work, logs);
saveLogs(workFile, logs);
File backup = new File(file.getParentFile(), file.getName() + ".bak");
if (file.renameTo(backup))
{
if (!work.renameTo(file))
{
backup.renameTo(file);
}
}
try
{
String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \"");
System.out.print(out);
out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + ".bak | sort | sha1sum \"");
System.out.println(out);
}
catch (CmdExecException exception)
{
exception.printStackTrace();
}
}
/**
* Sort error log.
*
* @param file
* the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void sortErrorLog(final File file) throws IOException
{
File work = new File(file.getParent(), file.getName() + ".tmp");
Logs logs = loadErrorLog(file);
logs.sortByDatetime();
saveLogs(work, logs);
File backup = new File(file.getParentFile(), file.getName() + ".bak");
if (file.renameTo(backup))
{
if (!work.renameTo(file))
if (!workFile.renameTo(file))
{
backup.renameTo(file);
}
}
// Check.
try
{
String out = CmdExecUtils.run("/bin/bash -c \"zcat " + file.getAbsolutePath() + "| sort | sha1sum \"");

6
src/org/april/logar/cli/LogarCLI.java
View file

@ -56,8 +56,8 @@ public final class LogarCLI
message.appendln(" logar [ -v | -version | --version ]");
message.appendln(" logar anonymize fileordirectory [maptable] anonymize ip and login");
message.appendln(" logar archive source target archive previous month");
message.appendln(" logar checksort fileordirectory check sort of an access log file");
message.appendln(" logar check fileordirectory census bad format line in log files");
message.appendln(" logar checksort fileordirectory check sort of an access log file");
message.appendln(" logar sort fileordirectory sort log files by datetime");
message.appendln(" logar testarchive source test archive");
@ -207,13 +207,13 @@ public final class LogarCLI
{
File source = new File(args[1]);
Logar.check(source);
Logar.checkLogFiles(source);
}
else if (isMatching(args, "checksort", "\\s*\\S+\\s*"))
{
File source = new File(args[1]);
Logar.checkSort(source);
Logar.checkSorts(source);
}
else if (isMatching(args, "sort", "\\s*\\S+\\s*"))
{

50
src/org/april/logar/util/Files.java
View file

@ -48,6 +48,31 @@ public class Files extends ArrayList<File>
super(initialCapacity);
}
/**
* Keep directories.
*
* @return the files
*/
public Files keepDirectoriesOnly()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (!file.isDirectory())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Removes the containing.
*
@ -75,6 +100,31 @@ public class Files extends ArrayList<File>
return result;
}
/**
* Removes the file type.
*
* @return the files
*/
public Files removeFileType()
{
Files result;
Iterator<File> iterator = iterator();
while (iterator.hasNext())
{
File file = iterator.next();
if (file.isFile())
{
iterator.remove();
}
}
result = this;
//
return result;
}
/**
* Removes the hidden.
*