statool/logar
statool/logar
1
0
Fork 0
Code Issues Pull requests Releases Activity

Improved line parser code.

Browse source
This commit is contained in:
Christian P. MOMON 2021-11-29 03:57:27 +01:00
parent 1178ab11a4
commit c59a00a695
11 changed files with 593 additions and 185 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
src/fr/devinsy/logar/app/Logar.java
View file

@ -35,7 +35,9 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.anonymizer.Anonymizer;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogFile;
import fr.devinsy.logar.app.log.LogParser;
import fr.devinsy.logar.app.log.LogType;
import fr.devinsy.logar.app.log.parser.AccessLogParser;
import fr.devinsy.logar.app.log.parser.LineParser;
import fr.devinsy.logar.stats.UserAgentStator;
import fr.devinsy.logar.util.Chrono;
import fr.devinsy.logar.util.Files;
@ -64,8 +66,9 @@ public final class Logar
*
* @param source
* the source
* @throws IOException
*/
public static void anonymize(final File source)
public static void anonymize(final File source) throws IOException
{
anonymize(source, null);
}
@ -77,8 +80,9 @@ public final class Logar
* the source
* @param mapFile
* the map file
* @throws IOException
*/
public static void anonymize(final File source, final File mapFile)
public static void anonymize(final File source, final File mapFile) throws IOException
{
if (source == null)
{
@ -150,6 +154,7 @@ public final class Logar
YearMonth targetYearMonth = YearMonth.now().minusMonths(1);
Stats counter = new Stats();
LineParser lineParser = new AccessLogParser();
for (File directory : Files.of(source).removeHidden().keepDirectoryType().sortByName())
{
String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString());
@ -179,7 +184,7 @@ public final class Logar
try
{
Log log = LogParser.parseAccessLog(line);
Log log = lineParser.parse(line);
counter.incSuccessLineCount();
if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
@ -268,6 +273,7 @@ public final class Logar
{
if ((!file.isDirectory()) && (file.getName().contains("error")))
{
LineParser lineParser = LogFile.getParser(file);
// logger.info(file.getName());
try
{
@ -279,7 +285,7 @@ public final class Logar
try
{
Log log = LogParser.parseErrorLog(line);
Log log = lineParser.parse(line);
counter.incSuccessLineCount();
if (YearMonth.from(log.getDatetime()).equals(targetYearMonth))
@ -325,8 +331,9 @@ public final class Logar
*
* @param file
* the source
* @throws IOException
*/
public static void checkLogFile(final File file)
public static void checkLogFile(final File file) throws IOException
{
if (file == null)
{
@ -339,7 +346,7 @@ public final class Logar
else
{
System.out.println("== Check parse log for [" + file.getName() + "]");
boolean isAccessFile = file.getName().contains("access");
LineParser parser = LogFile.getParser(file);
int lineCount = 0;
int badLineCount = 0;
@ -353,14 +360,7 @@ public final class Logar
try
{
if (isAccessFile)
{
LogParser.parseAccessLog(line).getDatetime();
}
else
{
LogParser.parseErrorLog(line).getDatetime();
}
parser.parse(line).getDatetime();
}
catch (IllegalArgumentException exception)
{
@ -393,8 +393,9 @@ public final class Logar
*
* @param source
* the source
* @throws IOException
*/
public static void checkLogFiles(final File source)
public static void checkLogFiles(final File source) throws IOException
{
if (source == null)
{
@ -464,7 +465,7 @@ public final class Logar
else
{
System.out.println("== Check sort for [" + file.getName() + "]");
boolean isAccessFile = file.getName().contains("access");
LineParser lineParser = LogFile.getParser(file);
LocalDateTime currentDate = null;
int lineCount = 0;
int badLineCount = 0;
@ -474,14 +475,7 @@ public final class Logar
String line = iterator.next();
lineCount += 1;
LocalDateTime date;
if (isAccessFile)
{
date = LogParser.parseAccessLog(line).getDatetime();
}
else
{
date = LogParser.parseErrorLog(line).getDatetime();
}
date = lineParser.parse(line).getDatetime();
if ((currentDate != null) && (date.isBefore(currentDate)))
{
@ -530,6 +524,7 @@ public final class Logar
System.err.println("== Extract userAgent for [" + file.getName() + "]");
try
{
LineParser lineParser = LogFile.getParser(file);
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
@ -537,7 +532,7 @@ public final class Logar
// System.out.println(line);
try
{
Log log = LogParser.parseAccessLog(line);
Log log = lineParser.parse(line);
StringList extract = new StringList();
if (options.getIp().isOn())
@ -582,6 +577,36 @@ public final class Logar
}
}
/**
* Identify.
*
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void identify(final File source) throws IOException
{
if (source == null)
{
System.out.println("Undefined source.");
}
else if (!source.exists())
{
System.out.println("Missing source to sort.");
}
else
{
Files files = FilesUtils.search(source, LOGFILE_PATTERN).removeHidden().sortByName();
for (File file : files)
{
String type = LogFile.getType(file);
System.out.println(StringUtils.rightPad(type, 20) + file.getName());
}
}
}
/**
* Sort.
*
@ -602,7 +627,7 @@ public final class Logar
}
else
{
Files files = FilesUtils.searchEndingWith(source, LOGFILE_PATTERN).removeHidden().sortByName();
Files files = FilesUtils.search(source, LOGFILE_PATTERN).removeHidden().sortByName();
for (File file : files)
{
@ -652,6 +677,7 @@ public final class Logar
int badLineCount = 0;
try
{
LineParser lineParser = LogFile.getParser(file);
UserAgentStator stator = new UserAgentStator();
LineIterator iterator = new LineIterator(file);
Chrono chrono = new Chrono().start();
@ -669,7 +695,7 @@ public final class Logar
try
{
Log log = LogParser.parseAccessLog(line);
Log log = lineParser.parse(line);
stator.putLog(log);
}
catch (IllegalArgumentException exception)
@ -724,8 +750,9 @@ public final class Logar
*
* @param source
* the source
* @throws IOException
*/
public static void testConcate(final File source)
public static void testConcate(final File source) throws IOException
{
Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log").sortByName();
@ -740,8 +767,9 @@ public final class Logar
*
* @param file
* the file
* @throws IOException
*/
public static void testConcateFile(final File file)
public static void testConcateFile(final File file) throws IOException
{
if (file == null)
{
@ -754,7 +782,7 @@ public final class Logar
else
{
System.out.println("== Test concate log for [" + file.getName() + "]");
boolean isAccessFile = file.getName().contains("access");
LineParser lineParser = LogFile.getParser(file);
int lineCount = 0;
int badLineCount = 0;
@ -768,18 +796,14 @@ public final class Logar
try
{
Log source;
Log target;
if (isAccessFile)
Log source = lineParser.parse(line);
Log target = new Log(source);
if (lineParser.getType() == LogType.ACCESS)
{
source = LogParser.parseAccessLog(line);
target = new Log(source);
target.concateAccessLog();
}
else
else if (lineParser.getType() == LogType.ERROR)
{
source = LogParser.parseErrorLog(line);
target = new Log(source);
target.concateErrorLog();
}

15
src/fr/devinsy/logar/app/anonymizer/Anonymizer.java
View file

@ -33,7 +33,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogParser;
import fr.devinsy.logar.app.log.LogFile;
import fr.devinsy.logar.app.log.LogType;
import fr.devinsy.logar.app.log.parser.LineParser;
import fr.devinsy.logar.util.LineIterator;
/**
@ -65,8 +67,9 @@ public final class Anonymizer
*
* @param source
* the source
* @throws IOException
*/
public void anonymize(final File source)
public void anonymize(final File source) throws IOException
{
if (source == null)
{
@ -83,7 +86,7 @@ public final class Anonymizer
else
{
System.out.println("== Anonymize log for [" + source.getName() + "]");
boolean isAccessFile = source.getName().contains("access");
LineParser lineParser = LogFile.getParser(source);
File target;
if (source.getName().endsWith(".log.gz"))
@ -106,10 +109,10 @@ public final class Anonymizer
try
{
Log log = lineParser.parse(line);
Log anon;
if (isAccessFile)
if (lineParser.getType() == LogType.ACCESS)
{
Log log = LogParser.parseAccessLog(line);
// logger.info("line={}", line);
// logger.info("log =[{}][{}][{}]", log.getIp(),
// log.getUser(), log.getDatetime());
@ -137,8 +140,6 @@ public final class Anonymizer
}
else
{
Log log = LogParser.parseErrorLog(line);
anon = anonymizeError(log);
}

214
src/fr/devinsy/logar/app/log/LogFile.java
View file

@ -31,6 +31,11 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.cmdexec.CmdExecException;
import fr.devinsy.cmdexec.CmdExecUtils;
import fr.devinsy.logar.app.log.parser.AccessLogParser;
import fr.devinsy.logar.app.log.parser.ApacheErrorLogParser;
import fr.devinsy.logar.app.log.parser.EmptyLogParser;
import fr.devinsy.logar.app.log.parser.LineParser;
import fr.devinsy.logar.app.log.parser.NginxErrorLogParser;
import fr.devinsy.logar.util.LineIterator;
/**
@ -48,7 +53,76 @@ public final class LogFile
}
/**
* Load access log.
* Gets the parser.
*
* @param file
* the file
* @return the parser
* @throws IOException
*/
public static LineParser getParser(final File file) throws IOException
{
LineParser result;
if (file.getName().contains("access"))
{
result = new AccessLogParser();
}
else if (file.getName().contains("error"))
{
String line = readFirstLine(file);
if (line == null)
{
result = new EmptyLogParser();
}
else if (line.startsWith("["))
{
result = new ApacheErrorLogParser();
}
else
{
result = new NginxErrorLogParser();
}
}
else
{
throw new IllegalArgumentException("Bad named file (missing access or error).");
}
//
return result;
}
/**
* Gets the type.
*
* @param file
* the file
* @return the type
* @throws IOException
*/
public static String getType(final File file) throws IOException
{
String result;
LineParser parser = getParser(file);
if (parser == null)
{
result = "*/*";
}
else
{
result = parser.getMime();
}
//
return result;
}
/**
* Parses the log file.
*
* @param file
* the file
@ -56,17 +130,18 @@ public final class LogFile
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadAccessLog(final File file) throws IOException
public static Logs parseLogFile(final File file) throws IOException
{
Logs result;
result = loadAccessLog(file, LogMode.NORMAL);
result = parseLogFile(file, LogMode.NORMAL);
//
return result;
}
/**
* Load access log.
* Parses the log file.
*
* @param file
* the file
@ -76,86 +151,7 @@ public final class LogFile
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadAccessLog(final File file, final LogMode mode) throws IOException
{
Logs result;
result = new Logs();
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
Log log = LogParser.parseAccessLog(line);
if (mode == LogMode.REDUCED)
{
log.reduce();
}
result.add(log);
}
//
return result;
}
/**
* Load error log.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadErrorLog(final File file) throws IOException
{
Logs result;
result = new Logs();
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
String line = iterator.next();
Log log = LogParser.parseErrorLog(line);
result.add(log);
}
//
return result;
}
/**
* Load log file.
*
* @param file
* the file
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadLogFile(final File file) throws IOException
{
Logs result;
result = loadLogFile(file, LogMode.NORMAL);
//
return result;
}
/**
* Load log file.
*
* @param file
* the file
* @param mode
* the mode
* @return the logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Logs loadLogFile(final File file, final LogMode mode) throws IOException
public static Logs parseLogFile(final File file, final LogMode mode) throws IOException
{
Logs result;
@ -165,17 +161,55 @@ public final class LogFile
}
else
{
if (file.getName().contains("access"))
LineParser lineParser = getParser(file);
result = new Logs();
LineIterator iterator = new LineIterator(file);
while (iterator.hasNext())
{
result = loadAccessLog(file, mode);
String line = iterator.next();
Log log = lineParser.parse(line);
if (mode == LogMode.REDUCED)
{
log.reduce();
}
else if (file.getName().contains("error"))
result.add(log);
}
}
//
return result;
}
/**
* Read first line.
*
* @param file
* the file
* @return the string
*/
private static String readFirstLine(final File file) throws IOException
{
result = loadErrorLog(file);
String result;
LineIterator iterator = null;
try
{
iterator = new LineIterator(file);
if (iterator.hasNext())
{
result = iterator.next();
}
else
{
throw new IllegalArgumentException("Bad named file (missing access or error).");
result = null;
}
}
finally
{
if (iterator != null)
{
iterator.close();
}
}
@ -232,7 +266,7 @@ public final class LogFile
{
File workFile = new File(file.getParent(), file.getName() + ".tmp");
Logs logs = loadLogFile(file, LogMode.REDUCED);
Logs logs = parseLogFile(file, LogMode.REDUCED);
logs.sortByDatetime();
saveLogs(workFile, logs);

29
src/fr/devinsy/logar/app/log/LogType.java Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log;
/**
* The Enum LogType.
*/
public enum LogType
{
ACCESS,
ERROR,
WILDCARD;
}

106
src/fr/devinsy/logar/app/log/LogParser.java → src/fr/devinsy/logar/app/log/parser/AccessLogParser.java
View file

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log;
package fr.devinsy.logar.app.log.parser;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
@ -28,25 +28,54 @@ import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogType;
/**
* The Class LogParser.
* The Class NginxAccessLogParser.
*/
public final class LogParser
public final class AccessLogParser implements LineParser
{
private static Logger logger = LoggerFactory.getLogger(LogParser.class);
private static Logger logger = LoggerFactory.getLogger(AccessLogParser.class);
public static Pattern NGINX_ACCESSLOG_LINE_PATTERN = Pattern.compile(
public static Pattern COMBINED_ACCESSLOG_LINE_PATTERN = Pattern.compile(
"^(?<remoteAddress>[a-zA-F0-9\\\\:\\\\.]+) - (?<remoteUser>[^\\[]+) \\[(?<datetime>[^\\]]+)\\] \"(?<request>[^\"]*)\" (?<status>\\d+) (?<bodyBytesSent>\\d+) \"(?<referer>[^\"]*)\" \"(?<userAgent>[^\"]*)\".*$");
public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?<datetime>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
/**
* Instantiates a new log parser.
*/
private LogParser()
public AccessLogParser()
{
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#type()
*/
@Override
public String getMime()
{
String result;
result = "Access/*";
//
return result;
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#getType()
*/
@Override
public LogType getType()
{
LogType result;
result = LogType.ACCESS;
//
return result;
}
/**
* From access log.
*
@ -58,13 +87,14 @@ public final class LogParser
* '"$request" $status $body_bytes_sent ' '"$http_referer"
* "$http_user_agent"';
*/
public static Log parseAccessLog(final String line)
@Override
public Log parse(final String line)
{
Log result;
try
{
Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line);
Matcher matcher = COMBINED_ACCESSLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String dateTimeValue = matcher.group("datetime");
@ -94,60 +124,4 @@ public final class LogParser
//
return result;
}
/**
* Parses the access log light.
*
* @param line
* the line
* @return the log
*/
public static Log parseAccessLogLight(final String line)
{
Log result;
result = parseAccessLog(line);
result.reduce();
//
return result;
}
/**
* From error log.
*
* @param line
* the line
* @return the log
*/
public static Log parseErrorLog(final String line)
{
Log result;
try
{
Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String value = matcher.group("datetime");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
result = new Log(line, date);
result.setDatetimeValue(matcher.group("datetime"));
result.setLevel(matcher.group("level"));
result.setMessage(matcher.group("message"));
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line, exception);
}
//
return result;
}
}

113
src/fr/devinsy/logar/app/log/parser/ApacheErrorLogParser.java Normal file
View file

@ -0,0 +1,113 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log.parser;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogType;
/**
* The Class ApacheErrorLogParser.
*/
public final class ApacheErrorLogParser implements LineParser
{
private static Logger logger = LoggerFactory.getLogger(ApacheErrorLogParser.class);
public static Pattern APACHE_ERRORLOG_LINE_PATTERN = Pattern.compile("^\\[(?<datetime>[^\\]]+)\\]\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
/**
* Instantiates a new nginx error log parser.
*/
public ApacheErrorLogParser()
{
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#type()
*/
@Override
public String getMime()
{
String result;
result = "Error/Apache";
//
return result;
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#getType()
*/
@Override
public LogType getType()
{
LogType result;
result = LogType.ERROR;
//
return result;
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#parse(java.lang.String)
*/
@Override
public Log parse(final String line)
{
Log result;
try
{
Matcher matcher = APACHE_ERRORLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String value = matcher.group("datetime");
// Example: Mon Nov 22 06:15:13.773450 2021
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss.SSSSSS yyyy").withLocale(Locale.ENGLISH));
result = new Log(line, date);
result.setDatetimeValue(matcher.group("datetime"));
result.setLevel(matcher.group("level"));
result.setMessage(matcher.group("message"));
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line, exception);
}
//
return result;
}
}

82
src/fr/devinsy/logar/app/log/parser/EmptyLogParser.java Normal file
View file

@ -0,0 +1,82 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log.parser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogType;
/**
* The Class EmptyLogParser.
*/
public final class EmptyLogParser implements LineParser
{
private static Logger logger = LoggerFactory.getLogger(EmptyLogParser.class);
/**
* Instantiates a new empty log parser.
*/
public EmptyLogParser()
{
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#type()
*/
@Override
public String getMime()
{
String result;
result = "*/*";
//
return result;
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#getType()
*/
@Override
public LogType getType()
{
LogType result;
result = LogType.WILDCARD;
//
return result;
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#parse(java.lang.String)
*/
@Override
public Log parse(final String line)
{
Log result;
result = null;
//
return result;
}
}

34
src/fr/devinsy/logar/app/log/parser/LineParser.java Normal file
View file

@ -0,0 +1,34 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log.parser;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogType;
/**
* The Interface LineParser.
*/
public interface LineParser
{
String getMime();
LogType getType();
Log parse(String line);
}

112
src/fr/devinsy/logar/app/log/parser/NginxErrorLogParser.java Normal file
View file

@ -0,0 +1,112 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
* Logar is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Logar is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.app.log.parser;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.logar.app.log.Log;
import fr.devinsy.logar.app.log.LogType;
/**
* The Class NginxErrorLogParser.
*/
public final class NginxErrorLogParser implements LineParser
{
private static Logger logger = LoggerFactory.getLogger(NginxErrorLogParser.class);
public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?<datetime>\\S+\\s\\S+)\\s\\[(?<level>[^\\]]*)\\]\\s(?<message>.*)$");
/**
* Instantiates a new nginx error log parser.
*/
public NginxErrorLogParser()
{
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#type()
*/
@Override
public String getMime()
{
String result;
result = "Error/Nginx";
//
return result;
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#getType()
*/
@Override
public LogType getType()
{
LogType result;
result = LogType.ERROR;
//
return result;
}
/* (non-Javadoc)
* @see fr.devinsy.logar.app.log.parser.LineParser#parse(java.lang.String)
*/
@Override
public Log parse(final String line)
{
Log result;
try
{
Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line);
if (matcher.matches())
{
String value = matcher.group("datetime");
LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH));
result = new Log(line, date);
result.setDatetimeValue(matcher.group("datetime"));
result.setLevel(matcher.group("level"));
result.setMessage(matcher.group("message"));
}
else
{
throw new IllegalArgumentException("Bad line format: " + line);
}
}
catch (DateTimeParseException exception)
{
throw new IllegalArgumentException("Bad line format (date): " + line, exception);
}
//
return result;
}
}

7
src/fr/devinsy/logar/cli/LogarCLI.java
View file

@ -61,6 +61,7 @@ public final class LogarCLI
message.appendln(" logar check <fileordirectory> check line format in log files");
message.appendln(" logar checksort <fileordirectory> check sort in log files");
message.appendln(" logar extract <fields> <fileordirectory> extract one or more fields (ip,user,datetime,useragent) from log files");
message.appendln(" logar identify <fileordirectory> display type of log files");
message.appendln(" logar sort <fileordirectory> sort log files by datetime");
message.appendln(" logar testconcate <fileordirectory> test line concate in log files");
@ -232,6 +233,12 @@ public final class LogarCLI
Logar.extract(source, options);
}
else if (isMatching(args, "identify", "\\s*\\S+\\s*"))
{
File source = new File(args[1]);
Logar.identify(source);
}
else if (isMatching(args, "sort", "\\s*\\S+\\s*"))
{
File source = new File(args[1]);

4
src/fr/devinsy/logar/stats/UserAgent.java
View file

@ -33,10 +33,8 @@ public final class UserAgent
private long ipLinkCount;
/**
* Instantiates a new user agent stat.
* Instantiates a new user agent.
*
* @param ip
* the ip
* @param userAgent
* the user agent
*/