From c59a00a6958adc8ceeddcb0807efc8af4f60e240 Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Mon, 29 Nov 2021 03:57:27 +0100 Subject: [PATCH] Improved line parser code. --- src/fr/devinsy/logar/app/Logar.java | 102 ++++++---- .../logar/app/anonymizer/Anonymizer.java | 15 +- src/fr/devinsy/logar/app/log/LogFile.java | 174 +++++++++++------- src/fr/devinsy/logar/app/log/LogType.java | 29 +++ .../AccessLogParser.java} | 106 ++++------- .../app/log/parser/ApacheErrorLogParser.java | 113 ++++++++++++ .../logar/app/log/parser/EmptyLogParser.java | 82 +++++++++ .../logar/app/log/parser/LineParser.java | 34 ++++ .../app/log/parser/NginxErrorLogParser.java | 112 +++++++++++ src/fr/devinsy/logar/cli/LogarCLI.java | 7 + src/fr/devinsy/logar/stats/UserAgent.java | 4 +- 11 files changed, 593 insertions(+), 185 deletions(-) create mode 100644 src/fr/devinsy/logar/app/log/LogType.java rename src/fr/devinsy/logar/app/log/{LogParser.java => parser/AccessLogParser.java} (61%) create mode 100644 src/fr/devinsy/logar/app/log/parser/ApacheErrorLogParser.java create mode 100644 src/fr/devinsy/logar/app/log/parser/EmptyLogParser.java create mode 100644 src/fr/devinsy/logar/app/log/parser/LineParser.java create mode 100644 src/fr/devinsy/logar/app/log/parser/NginxErrorLogParser.java diff --git a/src/fr/devinsy/logar/app/Logar.java b/src/fr/devinsy/logar/app/Logar.java index 5c005e6..a1b72b5 100644 --- a/src/fr/devinsy/logar/app/Logar.java +++ b/src/fr/devinsy/logar/app/Logar.java @@ -35,7 +35,9 @@ import org.slf4j.LoggerFactory; import fr.devinsy.logar.app.anonymizer.Anonymizer; import fr.devinsy.logar.app.log.Log; import fr.devinsy.logar.app.log.LogFile; -import fr.devinsy.logar.app.log.LogParser; +import fr.devinsy.logar.app.log.LogType; +import fr.devinsy.logar.app.log.parser.AccessLogParser; +import fr.devinsy.logar.app.log.parser.LineParser; import fr.devinsy.logar.stats.UserAgentStator; import fr.devinsy.logar.util.Chrono; import fr.devinsy.logar.util.Files; @@ -64,8 +66,9 @@ public final class Logar * * @param source * the source + * @throws IOException */ - public static void anonymize(final File source) + public static void anonymize(final File source) throws IOException { anonymize(source, null); } @@ -77,8 +80,9 @@ public final class Logar * the source * @param mapFile * the map file + * @throws IOException */ - public static void anonymize(final File source, final File mapFile) + public static void anonymize(final File source, final File mapFile) throws IOException { if (source == null) { @@ -150,6 +154,7 @@ public final class Logar YearMonth targetYearMonth = YearMonth.now().minusMonths(1); Stats counter = new Stats(); + LineParser lineParser = new AccessLogParser(); for (File directory : Files.of(source).removeHidden().keepDirectoryType().sortByName()) { String targetFileName = String.format("%s-access-%s.log.gz", directory.getName(), targetYearMonth.toString()); @@ -179,7 +184,7 @@ public final class Logar try { - Log log = LogParser.parseAccessLog(line); + Log log = lineParser.parse(line); counter.incSuccessLineCount(); if (YearMonth.from(log.getDatetime()).equals(targetYearMonth)) @@ -268,6 +273,7 @@ public final class Logar { if ((!file.isDirectory()) && (file.getName().contains("error"))) { + LineParser lineParser = LogFile.getParser(file); // logger.info(file.getName()); try { @@ -279,7 +285,7 @@ public final class Logar try { - Log log = LogParser.parseErrorLog(line); + Log log = lineParser.parse(line); counter.incSuccessLineCount(); if (YearMonth.from(log.getDatetime()).equals(targetYearMonth)) @@ -325,8 +331,9 @@ public final class Logar * * @param file * the source + * @throws IOException */ - public static void checkLogFile(final File file) + public static void checkLogFile(final File file) throws IOException { if (file == null) { @@ -339,7 +346,7 @@ public final class Logar else { System.out.println("== Check parse log for [" + file.getName() + "]"); - boolean isAccessFile = file.getName().contains("access"); + LineParser parser = LogFile.getParser(file); int lineCount = 0; int badLineCount = 0; @@ -353,14 +360,7 @@ public final class Logar try { - if (isAccessFile) - { - LogParser.parseAccessLog(line).getDatetime(); - } - else - { - LogParser.parseErrorLog(line).getDatetime(); - } + parser.parse(line).getDatetime(); } catch (IllegalArgumentException exception) { @@ -393,8 +393,9 @@ public final class Logar * * @param source * the source + * @throws IOException */ - public static void checkLogFiles(final File source) + public static void checkLogFiles(final File source) throws IOException { if (source == null) { @@ -464,7 +465,7 @@ public final class Logar else { System.out.println("== Check sort for [" + file.getName() + "]"); - boolean isAccessFile = file.getName().contains("access"); + LineParser lineParser = LogFile.getParser(file); LocalDateTime currentDate = null; int lineCount = 0; int badLineCount = 0; @@ -474,14 +475,7 @@ public final class Logar String line = iterator.next(); lineCount += 1; LocalDateTime date; - if (isAccessFile) - { - date = LogParser.parseAccessLog(line).getDatetime(); - } - else - { - date = LogParser.parseErrorLog(line).getDatetime(); - } + date = lineParser.parse(line).getDatetime(); if ((currentDate != null) && (date.isBefore(currentDate))) { @@ -530,6 +524,7 @@ public final class Logar System.err.println("== Extract userAgent for [" + file.getName() + "]"); try { + LineParser lineParser = LogFile.getParser(file); LineIterator iterator = new LineIterator(file); while (iterator.hasNext()) { @@ -537,7 +532,7 @@ public final class Logar // System.out.println(line); try { - Log log = LogParser.parseAccessLog(line); + Log log = lineParser.parse(line); StringList extract = new StringList(); if (options.getIp().isOn()) @@ -582,6 +577,36 @@ public final class Logar } } + /** + * Identify. + * + * @param source + * the source + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public static void identify(final File source) throws IOException + { + if (source == null) + { + System.out.println("Undefined source."); + } + else if (!source.exists()) + { + System.out.println("Missing source to sort."); + } + else + { + Files files = FilesUtils.search(source, LOGFILE_PATTERN).removeHidden().sortByName(); + for (File file : files) + { + String type = LogFile.getType(file); + + System.out.println(StringUtils.rightPad(type, 20) + file.getName()); + } + } + } + /** * Sort. * @@ -602,7 +627,7 @@ public final class Logar } else { - Files files = FilesUtils.searchEndingWith(source, LOGFILE_PATTERN).removeHidden().sortByName(); + Files files = FilesUtils.search(source, LOGFILE_PATTERN).removeHidden().sortByName(); for (File file : files) { @@ -652,6 +677,7 @@ public final class Logar int badLineCount = 0; try { + LineParser lineParser = LogFile.getParser(file); UserAgentStator stator = new UserAgentStator(); LineIterator iterator = new LineIterator(file); Chrono chrono = new Chrono().start(); @@ -669,7 +695,7 @@ public final class Logar try { - Log log = LogParser.parseAccessLog(line); + Log log = lineParser.parse(line); stator.putLog(log); } catch (IllegalArgumentException exception) @@ -724,8 +750,9 @@ public final class Logar * * @param source * the source + * @throws IOException */ - public static void testConcate(final File source) + public static void testConcate(final File source) throws IOException { Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log").sortByName(); @@ -740,8 +767,9 @@ public final class Logar * * @param file * the file + * @throws IOException */ - public static void testConcateFile(final File file) + public static void testConcateFile(final File file) throws IOException { if (file == null) { @@ -754,7 +782,7 @@ public final class Logar else { System.out.println("== Test concate log for [" + file.getName() + "]"); - boolean isAccessFile = file.getName().contains("access"); + LineParser lineParser = LogFile.getParser(file); int lineCount = 0; int badLineCount = 0; @@ -768,18 +796,14 @@ public final class Logar try { - Log source; - Log target; - if (isAccessFile) + Log source = lineParser.parse(line); + Log target = new Log(source); + if (lineParser.getType() == LogType.ACCESS) { - source = LogParser.parseAccessLog(line); - target = new Log(source); target.concateAccessLog(); } - else + else if (lineParser.getType() == LogType.ERROR) { - source = LogParser.parseErrorLog(line); - target = new Log(source); target.concateErrorLog(); } diff --git a/src/fr/devinsy/logar/app/anonymizer/Anonymizer.java b/src/fr/devinsy/logar/app/anonymizer/Anonymizer.java index 88bb033..1e2c939 100644 --- a/src/fr/devinsy/logar/app/anonymizer/Anonymizer.java +++ b/src/fr/devinsy/logar/app/anonymizer/Anonymizer.java @@ -33,7 +33,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import fr.devinsy.logar.app.log.Log; -import fr.devinsy.logar.app.log.LogParser; +import fr.devinsy.logar.app.log.LogFile; +import fr.devinsy.logar.app.log.LogType; +import fr.devinsy.logar.app.log.parser.LineParser; import fr.devinsy.logar.util.LineIterator; /** @@ -65,8 +67,9 @@ public final class Anonymizer * * @param source * the source + * @throws IOException */ - public void anonymize(final File source) + public void anonymize(final File source) throws IOException { if (source == null) { @@ -83,7 +86,7 @@ public final class Anonymizer else { System.out.println("== Anonymize log for [" + source.getName() + "]"); - boolean isAccessFile = source.getName().contains("access"); + LineParser lineParser = LogFile.getParser(source); File target; if (source.getName().endsWith(".log.gz")) @@ -106,10 +109,10 @@ public final class Anonymizer try { + Log log = lineParser.parse(line); Log anon; - if (isAccessFile) + if (lineParser.getType() == LogType.ACCESS) { - Log log = LogParser.parseAccessLog(line); // logger.info("line={}", line); // logger.info("log =[{}][{}][{}]", log.getIp(), // log.getUser(), log.getDatetime()); @@ -137,8 +140,6 @@ public final class Anonymizer } else { - Log log = LogParser.parseErrorLog(line); - anon = anonymizeError(log); } diff --git a/src/fr/devinsy/logar/app/log/LogFile.java b/src/fr/devinsy/logar/app/log/LogFile.java index f24bfbc..c6ab59a 100644 --- a/src/fr/devinsy/logar/app/log/LogFile.java +++ b/src/fr/devinsy/logar/app/log/LogFile.java @@ -31,6 +31,11 @@ import org.slf4j.LoggerFactory; import fr.devinsy.cmdexec.CmdExecException; import fr.devinsy.cmdexec.CmdExecUtils; +import fr.devinsy.logar.app.log.parser.AccessLogParser; +import fr.devinsy.logar.app.log.parser.ApacheErrorLogParser; +import fr.devinsy.logar.app.log.parser.EmptyLogParser; +import fr.devinsy.logar.app.log.parser.LineParser; +import fr.devinsy.logar.app.log.parser.NginxErrorLogParser; import fr.devinsy.logar.util.LineIterator; /** @@ -48,77 +53,41 @@ public final class LogFile } /** - * Load access log. + * Gets the parser. * * @param file * the file - * @return the logs + * @return the parser * @throws IOException - * Signals that an I/O exception has occurred. */ - public static Logs loadAccessLog(final File file) throws IOException + public static LineParser getParser(final File file) throws IOException { - Logs result; + LineParser result; - result = loadAccessLog(file, LogMode.NORMAL); - - return result; - } - - /** - * Load access log. - * - * @param file - * the file - * @param mode - * the mode - * @return the logs - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public static Logs loadAccessLog(final File file, final LogMode mode) throws IOException - { - Logs result; - - result = new Logs(); - - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) + if (file.getName().contains("access")) { - String line = iterator.next(); - Log log = LogParser.parseAccessLog(line); - if (mode == LogMode.REDUCED) + result = new AccessLogParser(); + } + else if (file.getName().contains("error")) + { + String line = readFirstLine(file); + + if (line == null) { - log.reduce(); + result = new EmptyLogParser(); + } + else if (line.startsWith("[")) + { + result = new ApacheErrorLogParser(); + } + else + { + result = new NginxErrorLogParser(); } - result.add(log); } - - // - return result; - } - - /** - * Load error log. - * - * @param file - * the file - * @return the logs - * @throws IOException - * Signals that an I/O exception has occurred. - */ - public static Logs loadErrorLog(final File file) throws IOException - { - Logs result; - - result = new Logs(); - - LineIterator iterator = new LineIterator(file); - while (iterator.hasNext()) + else { - String line = iterator.next(); - Log log = LogParser.parseErrorLog(line); - result.add(log); + throw new IllegalArgumentException("Bad named file (missing access or error)."); } // @@ -126,7 +95,34 @@ public final class LogFile } /** - * Load log file. + * Gets the type. + * + * @param file + * the file + * @return the type + * @throws IOException + */ + public static String getType(final File file) throws IOException + { + String result; + + LineParser parser = getParser(file); + + if (parser == null) + { + result = "*/*"; + } + else + { + result = parser.getMime(); + } + + // + return result; + } + + /** + * Parses the log file. * * @param file * the file @@ -134,18 +130,18 @@ public final class LogFile * @throws IOException * Signals that an I/O exception has occurred. */ - public static Logs loadLogFile(final File file) throws IOException + public static Logs parseLogFile(final File file) throws IOException { Logs result; - result = loadLogFile(file, LogMode.NORMAL); + result = parseLogFile(file, LogMode.NORMAL); // return result; } /** - * Load log file. + * Parses the log file. * * @param file * the file @@ -155,7 +151,7 @@ public final class LogFile * @throws IOException * Signals that an I/O exception has occurred. */ - public static Logs loadLogFile(final File file, final LogMode mode) throws IOException + public static Logs parseLogFile(final File file, final LogMode mode) throws IOException { Logs result; @@ -165,17 +161,55 @@ public final class LogFile } else { - if (file.getName().contains("access")) + LineParser lineParser = getParser(file); + + result = new Logs(); + LineIterator iterator = new LineIterator(file); + while (iterator.hasNext()) { - result = loadAccessLog(file, mode); + String line = iterator.next(); + Log log = lineParser.parse(line); + if (mode == LogMode.REDUCED) + { + log.reduce(); + } + result.add(log); } - else if (file.getName().contains("error")) + } + + // + return result; + } + + /** + * Read first line. + * + * @param file + * the file + * @return the string + */ + private static String readFirstLine(final File file) throws IOException + { + String result; + + LineIterator iterator = null; + try + { + iterator = new LineIterator(file); + if (iterator.hasNext()) { - result = loadErrorLog(file); + result = iterator.next(); } else { - throw new IllegalArgumentException("Bad named file (missing access or error)."); + result = null; + } + } + finally + { + if (iterator != null) + { + iterator.close(); } } @@ -232,7 +266,7 @@ public final class LogFile { File workFile = new File(file.getParent(), file.getName() + ".tmp"); - Logs logs = loadLogFile(file, LogMode.REDUCED); + Logs logs = parseLogFile(file, LogMode.REDUCED); logs.sortByDatetime(); saveLogs(workFile, logs); diff --git a/src/fr/devinsy/logar/app/log/LogType.java b/src/fr/devinsy/logar/app/log/LogType.java new file mode 100644 index 0000000..6bb39d9 --- /dev/null +++ b/src/fr/devinsy/logar/app/log/LogType.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package fr.devinsy.logar.app.log; + +/** + * The Enum LogType. + */ +public enum LogType +{ + ACCESS, + ERROR, + WILDCARD; +} diff --git a/src/fr/devinsy/logar/app/log/LogParser.java b/src/fr/devinsy/logar/app/log/parser/AccessLogParser.java similarity index 61% rename from src/fr/devinsy/logar/app/log/LogParser.java rename to src/fr/devinsy/logar/app/log/parser/AccessLogParser.java index cfe8ef3..be3751e 100644 --- a/src/fr/devinsy/logar/app/log/LogParser.java +++ b/src/fr/devinsy/logar/app/log/parser/AccessLogParser.java @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with Logar. If not, see . */ -package fr.devinsy.logar.app.log; +package fr.devinsy.logar.app.log.parser; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -28,25 +28,54 @@ import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import fr.devinsy.logar.app.log.Log; +import fr.devinsy.logar.app.log.LogType; + /** - * The Class LogParser. + * The Class NginxAccessLogParser. */ -public final class LogParser +public final class AccessLogParser implements LineParser { - private static Logger logger = LoggerFactory.getLogger(LogParser.class); + private static Logger logger = LoggerFactory.getLogger(AccessLogParser.class); - public static Pattern NGINX_ACCESSLOG_LINE_PATTERN = Pattern.compile( + public static Pattern COMBINED_ACCESSLOG_LINE_PATTERN = Pattern.compile( "^(?[a-zA-F0-9\\\\:\\\\.]+) - (?[^\\[]+) \\[(?[^\\]]+)\\] \"(?[^\"]*)\" (?\\d+) (?\\d+) \"(?[^\"]*)\" \"(?[^\"]*)\".*$"); - public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?\\S+\\s\\S+)\\s\\[(?[^\\]]*)\\]\\s(?.*)$"); - /** * Instantiates a new log parser. */ - private LogParser() + public AccessLogParser() { } + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#type() + */ + @Override + public String getMime() + { + String result; + + result = "Access/*"; + + // + return result; + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#getType() + */ + @Override + public LogType getType() + { + LogType result; + + result = LogType.ACCESS; + + // + return result; + } + /** * From access log. * @@ -58,13 +87,14 @@ public final class LogParser * '"$request" $status $body_bytes_sent ' '"$http_referer" * "$http_user_agent"'; */ - public static Log parseAccessLog(final String line) + @Override + public Log parse(final String line) { Log result; try { - Matcher matcher = NGINX_ACCESSLOG_LINE_PATTERN.matcher(line); + Matcher matcher = COMBINED_ACCESSLOG_LINE_PATTERN.matcher(line); if (matcher.matches()) { String dateTimeValue = matcher.group("datetime"); @@ -94,60 +124,4 @@ public final class LogParser // return result; } - - /** - * Parses the access log light. - * - * @param line - * the line - * @return the log - */ - public static Log parseAccessLogLight(final String line) - { - Log result; - - result = parseAccessLog(line); - result.reduce(); - - // - return result; - } - - /** - * From error log. - * - * @param line - * the line - * @return the log - */ - public static Log parseErrorLog(final String line) - { - Log result; - - try - { - Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line); - if (matcher.matches()) - { - String value = matcher.group("datetime"); - LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)); - - result = new Log(line, date); - result.setDatetimeValue(matcher.group("datetime")); - result.setLevel(matcher.group("level")); - result.setMessage(matcher.group("message")); - } - else - { - throw new IllegalArgumentException("Bad line format: " + line); - } - } - catch (DateTimeParseException exception) - { - throw new IllegalArgumentException("Bad line format (date): " + line, exception); - } - - // - return result; - } } diff --git a/src/fr/devinsy/logar/app/log/parser/ApacheErrorLogParser.java b/src/fr/devinsy/logar/app/log/parser/ApacheErrorLogParser.java new file mode 100644 index 0000000..3c4a07f --- /dev/null +++ b/src/fr/devinsy/logar/app/log/parser/ApacheErrorLogParser.java @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package fr.devinsy.logar.app.log.parser; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.logar.app.log.Log; +import fr.devinsy.logar.app.log.LogType; + +/** + * The Class ApacheErrorLogParser. + */ +public final class ApacheErrorLogParser implements LineParser +{ + private static Logger logger = LoggerFactory.getLogger(ApacheErrorLogParser.class); + + public static Pattern APACHE_ERRORLOG_LINE_PATTERN = Pattern.compile("^\\[(?[^\\]]+)\\]\\s\\[(?[^\\]]*)\\]\\s(?.*)$"); + + /** + * Instantiates a new nginx error log parser. + */ + public ApacheErrorLogParser() + { + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#type() + */ + @Override + public String getMime() + { + String result; + + result = "Error/Apache"; + + // + return result; + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#getType() + */ + @Override + public LogType getType() + { + LogType result; + + result = LogType.ERROR; + + // + return result; + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#parse(java.lang.String) + */ + @Override + public Log parse(final String line) + { + Log result; + + try + { + Matcher matcher = APACHE_ERRORLOG_LINE_PATTERN.matcher(line); + if (matcher.matches()) + { + String value = matcher.group("datetime"); + // Example: Mon Nov 22 06:15:13.773450 2021 + LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("EEE MMM dd HH:mm:ss.SSSSSS yyyy").withLocale(Locale.ENGLISH)); + + result = new Log(line, date); + result.setDatetimeValue(matcher.group("datetime")); + result.setLevel(matcher.group("level")); + result.setMessage(matcher.group("message")); + } + else + { + throw new IllegalArgumentException("Bad line format: " + line); + } + } + catch (DateTimeParseException exception) + { + throw new IllegalArgumentException("Bad line format (date): " + line, exception); + } + + // + return result; + } +} diff --git a/src/fr/devinsy/logar/app/log/parser/EmptyLogParser.java b/src/fr/devinsy/logar/app/log/parser/EmptyLogParser.java new file mode 100644 index 0000000..da2a458 --- /dev/null +++ b/src/fr/devinsy/logar/app/log/parser/EmptyLogParser.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package fr.devinsy.logar.app.log.parser; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.logar.app.log.Log; +import fr.devinsy.logar.app.log.LogType; + +/** + * The Class EmptyLogParser. + */ +public final class EmptyLogParser implements LineParser +{ + private static Logger logger = LoggerFactory.getLogger(EmptyLogParser.class); + + /** + * Instantiates a new empty log parser. + */ + public EmptyLogParser() + { + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#type() + */ + @Override + public String getMime() + { + String result; + + result = "*/*"; + + // + return result; + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#getType() + */ + @Override + public LogType getType() + { + LogType result; + + result = LogType.WILDCARD; + + // + return result; + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#parse(java.lang.String) + */ + @Override + public Log parse(final String line) + { + Log result; + + result = null; + + // + return result; + } +} diff --git a/src/fr/devinsy/logar/app/log/parser/LineParser.java b/src/fr/devinsy/logar/app/log/parser/LineParser.java new file mode 100644 index 0000000..1dd70ee --- /dev/null +++ b/src/fr/devinsy/logar/app/log/parser/LineParser.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package fr.devinsy.logar.app.log.parser; + +import fr.devinsy.logar.app.log.Log; +import fr.devinsy.logar.app.log.LogType; + +/** + * The Interface LineParser. + */ +public interface LineParser +{ + String getMime(); + + LogType getType(); + + Log parse(String line); +} diff --git a/src/fr/devinsy/logar/app/log/parser/NginxErrorLogParser.java b/src/fr/devinsy/logar/app/log/parser/NginxErrorLogParser.java new file mode 100644 index 0000000..156d240 --- /dev/null +++ b/src/fr/devinsy/logar/app/log/parser/NginxErrorLogParser.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of Logar, simple tool to manage http log files. + * + * Logar is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * Logar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Logar. If not, see . + */ +package fr.devinsy.logar.app.log.parser; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.logar.app.log.Log; +import fr.devinsy.logar.app.log.LogType; + +/** + * The Class NginxErrorLogParser. + */ +public final class NginxErrorLogParser implements LineParser +{ + private static Logger logger = LoggerFactory.getLogger(NginxErrorLogParser.class); + + public static Pattern NGINX_ERRORLOG_LINE_PATTERN = Pattern.compile("^(?\\S+\\s\\S+)\\s\\[(?[^\\]]*)\\]\\s(?.*)$"); + + /** + * Instantiates a new nginx error log parser. + */ + public NginxErrorLogParser() + { + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#type() + */ + @Override + public String getMime() + { + String result; + + result = "Error/Nginx"; + + // + return result; + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#getType() + */ + @Override + public LogType getType() + { + LogType result; + + result = LogType.ERROR; + + // + return result; + } + + /* (non-Javadoc) + * @see fr.devinsy.logar.app.log.parser.LineParser#parse(java.lang.String) + */ + @Override + public Log parse(final String line) + { + Log result; + + try + { + Matcher matcher = NGINX_ERRORLOG_LINE_PATTERN.matcher(line); + if (matcher.matches()) + { + String value = matcher.group("datetime"); + LocalDateTime date = LocalDateTime.parse(value, DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss").withLocale(Locale.ENGLISH)); + + result = new Log(line, date); + result.setDatetimeValue(matcher.group("datetime")); + result.setLevel(matcher.group("level")); + result.setMessage(matcher.group("message")); + } + else + { + throw new IllegalArgumentException("Bad line format: " + line); + } + } + catch (DateTimeParseException exception) + { + throw new IllegalArgumentException("Bad line format (date): " + line, exception); + } + + // + return result; + } +} diff --git a/src/fr/devinsy/logar/cli/LogarCLI.java b/src/fr/devinsy/logar/cli/LogarCLI.java index 6b9cadc..63ba13c 100644 --- a/src/fr/devinsy/logar/cli/LogarCLI.java +++ b/src/fr/devinsy/logar/cli/LogarCLI.java @@ -61,6 +61,7 @@ public final class LogarCLI message.appendln(" logar check check line format in log files"); message.appendln(" logar checksort check sort in log files"); message.appendln(" logar extract extract one or more fields (ip,user,datetime,useragent) from log files"); + message.appendln(" logar identify display type of log files"); message.appendln(" logar sort sort log files by datetime"); message.appendln(" logar testconcate test line concate in log files"); @@ -232,6 +233,12 @@ public final class LogarCLI Logar.extract(source, options); } + else if (isMatching(args, "identify", "\\s*\\S+\\s*")) + { + File source = new File(args[1]); + + Logar.identify(source); + } else if (isMatching(args, "sort", "\\s*\\S+\\s*")) { File source = new File(args[1]); diff --git a/src/fr/devinsy/logar/stats/UserAgent.java b/src/fr/devinsy/logar/stats/UserAgent.java index 59954ba..35ac1d7 100644 --- a/src/fr/devinsy/logar/stats/UserAgent.java +++ b/src/fr/devinsy/logar/stats/UserAgent.java @@ -33,10 +33,8 @@ public final class UserAgent private long ipLinkCount; /** - * Instantiates a new user agent stat. + * Instantiates a new user agent. * - * @param ip - * the ip * @param userAgent * the user agent */