From be4635d8cb712f17e817fceeb4d5ab62d9460350 Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Sat, 5 Jun 2021 17:56:07 +0200 Subject: [PATCH] Added list and stat commands. --- .../statoolinfos/cli/StatoolInfosCLI.java | 130 ++++++- .../devinsy/statoolinfos/core/LogFilter.java | 67 ++++ .../statoolinfos/core/StatoolInfos.java | 316 +++++++++++++++++- .../metrics/http/HttpAccessLog.java | 36 +- .../metrics/http/HttpAccessLogAnalyzer.java | 12 +- .../metrics/http/VisitCounters.java | 44 ++- .../devinsy/statoolinfos/stats/ip/IpStat.java | 60 ++++ .../stats/ip/IpStatComparator.java | 139 ++++++++ .../statoolinfos/stats/ip/IpStatSet.java | 80 +++++ .../statoolinfos/stats/ip/IpStator.java | 71 ++++ .../statoolinfos/stats/ip/IpStats.java | 119 +++++++ .../stats/useragent/UserAgentStat.java | 75 +++++ .../useragent/UserAgentStatComparator.java | 139 ++++++++ .../stats/useragent/UserAgentStatSet.java | 85 +++++ .../stats/useragent/UserAgentStator.java | 78 +++++ .../stats/useragent/UserAgentStats.java | 119 +++++++ .../stats/visitor/VisitorStat.java | 93 ++++++ .../stats/visitor/VisitorStatComparator.java | 190 +++++++++++ .../stats/visitor/VisitorStatSet.java | 118 +++++++ .../stats/visitor/VisitorStator.java | 85 +++++ .../stats/visitor/VisitorStats.java | 149 +++++++++ src/fr/devinsy/statoolinfos/util/Chrono.java | 38 +++ .../metrics/http/VisitCountersTest.java | 24 +- 23 files changed, 2221 insertions(+), 46 deletions(-) create mode 100644 src/fr/devinsy/statoolinfos/core/LogFilter.java create mode 100644 src/fr/devinsy/statoolinfos/stats/ip/IpStat.java create mode 100644 src/fr/devinsy/statoolinfos/stats/ip/IpStatComparator.java create mode 100644 src/fr/devinsy/statoolinfos/stats/ip/IpStatSet.java create mode 100644 src/fr/devinsy/statoolinfos/stats/ip/IpStator.java create mode 100644 src/fr/devinsy/statoolinfos/stats/ip/IpStats.java create mode 100644 src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStat.java create mode 100644 src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStatComparator.java create mode 100644 src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStatSet.java create mode 100644 src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStator.java create mode 100644 src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStats.java create mode 100644 src/fr/devinsy/statoolinfos/stats/visitor/VisitorStat.java create mode 100644 src/fr/devinsy/statoolinfos/stats/visitor/VisitorStatComparator.java create mode 100644 src/fr/devinsy/statoolinfos/stats/visitor/VisitorStatSet.java create mode 100644 src/fr/devinsy/statoolinfos/stats/visitor/VisitorStator.java create mode 100644 src/fr/devinsy/statoolinfos/stats/visitor/VisitorStats.java diff --git a/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java b/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java index 430c5f8..4b6137b 100644 --- a/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java +++ b/src/fr/devinsy/statoolinfos/cli/StatoolInfosCLI.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import fr.devinsy.statoolinfos.core.LogFilter; import fr.devinsy.statoolinfos.core.StatoolInfos; import fr.devinsy.statoolinfos.util.BuildInformation; import fr.devinsy.statoolinfos.util.Chrono; @@ -103,13 +104,19 @@ public final class StatoolInfosCLI message.appendln("Usage:"); message.appendln(" statoolinfos [ -h | -help | --help ]"); message.appendln(" statoolinfos [ -v | -version | --version ]"); - message.appendln(" statoolinfos build build property files from conf and input"); - message.appendln(" statoolinfos clear remove property files from conf"); - message.appendln(" statoolinfos crawl crawl all file from conf and input"); - message.appendln(" statoolinfos format format in tiny way"); - message.appendln(" statoolinfos htmlize generate web pages from conf"); - message.appendln(" statoolinfos probe generate metrics files from conf"); - message.appendln(" statoolinfos tagdate update the file.datetime file"); + message.appendln(" statoolinfos build build property files from conf and input"); + message.appendln(" statoolinfos clear remove property files from conf"); + message.appendln(" statoolinfos crawl crawl all file from conf and input"); + message.appendln(" statoolinfos format format in tiny way"); + message.appendln(" statoolinfos htmlize generate web pages from conf"); + message.appendln(" statoolinfos list ip [-bot|-nobot] generate ip list from log file"); + message.appendln(" statoolinfos list ua [-bot|-nobot] generate user agent list from log file"); + message.appendln(" statoolinfos list visitors [-bot|-nobot] generate visitors (ip+ua) list from log file"); + message.appendln(" statoolinfos probe generate metrics files from conf"); + message.appendln(" statoolinfos stat ip [-bot|-nobot] generate stats about ip from log file"); + message.appendln(" statoolinfos stat ua [-bot|-nobot] generate stats about user agent from log file"); + message.appendln(" statoolinfos stat visitors [-bot|-nobot] generate stats about visitors (ip+ua) from log file"); + message.appendln(" statoolinfos tagdate update the file.datetime file"); System.out.println(message.toString()); } @@ -185,6 +192,38 @@ public final class StatoolInfosCLI return result; } + /** + * Parses the log filter option. + * + * @param source + * the source + * @return the log filter + */ + private static LogFilter parseLogFilterOption(final String source) + { + LogFilter result; + + if (StringUtils.equals(source, "-all")) + { + result = LogFilter.ALL; + } + else if (StringUtils.equals(source, "-bot")) + { + result = LogFilter.BOT; + } + else if (StringUtils.equals(source, "-nobot")) + { + result = LogFilter.NOBOT; + } + else + { + result = null; + } + + // + return result; + } + /** * * This method launch CLI. @@ -313,6 +352,45 @@ public final class StatoolInfosCLI } System.out.println(chrono.format()); } + else if (isMatching(args, "list", "ip", "\\s*\\S+\\s*")) + { + File source = new File(args[2]); + + StatoolInfos.listIps(source, LogFilter.ALL); + } + else if (isMatching(args, "list", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) + { + LogFilter filter = parseLogFilterOption(args[2]); + File source = new File(args[3]); + + StatoolInfos.listIps(source, filter); + } + else if (isMatching(args, "list", "(useragent|ua)", "\\s*\\S+\\s*")) + { + File source = new File(args[2]); + + StatoolInfos.listUserAgents(source, LogFilter.ALL); + } + else if (isMatching(args, "list", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) + { + LogFilter filter = parseLogFilterOption(args[2]); + File source = new File(args[3]); + + StatoolInfos.listUserAgents(source, filter); + } + else if (isMatching(args, "list", "visitors", "\\s*\\S+\\s*")) + { + File source = new File(args[2]); + + StatoolInfos.listVisitors(source, LogFilter.ALL); + } + else if (isMatching(args, "list", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) + { + LogFilter filter = parseLogFilterOption(args[2]); + File source = new File(args[3]); + + StatoolInfos.listVisitors(source, filter); + } else if (isMatching(args, "probe", "\\s*.+\\s*")) { Files inputs = convertPath(StringUtils.trim(args[1])); @@ -329,6 +407,44 @@ public final class StatoolInfosCLI } } } + else if (isMatching(args, "stat", "ip", "\\s*\\S+\\s*")) + { + File source = new File(args[2]); + + StatoolInfos.statIps(source, LogFilter.ALL); + } + else if (isMatching(args, "stat", "ip", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) + { + LogFilter filter = parseLogFilterOption(args[2]); + File source = new File(args[3]); + + StatoolInfos.statIps(source, filter); + } + else if (isMatching(args, "stat", "(useragent|ua)", "\\s*\\S+\\s*")) + { + File source = new File(args[2]); + StatoolInfos.statUserAgents(source, LogFilter.ALL); + } + else if (isMatching(args, "stat", "(useragent|ua)", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) + { + LogFilter filter = parseLogFilterOption(args[2]); + File source = new File(args[3]); + + StatoolInfos.statUserAgents(source, filter); + } + else if (isMatching(args, "stat", "visitors", "\\s*\\S+\\s*")) + { + File source = new File(args[2]); + + StatoolInfos.statVisitors(source, LogFilter.ALL); + } + else if (isMatching(args, "stat", "visitors", "(-all|-bot|-nobot)", "\\s*\\S+\\s*")) + { + LogFilter filter = parseLogFilterOption(args[2]); + File source = new File(args[3]); + + StatoolInfos.statVisitors(source, filter); + } else if (isMatching(args, "tagdate", "\\s*.+\\s*")) { Files inputs = FilesUtils.searchEndingWith(new File(args[1]), ".properties"); diff --git a/src/fr/devinsy/statoolinfos/core/LogFilter.java b/src/fr/devinsy/statoolinfos/core/LogFilter.java new file mode 100644 index 0000000..9c53f52 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/core/LogFilter.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2020 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.core; + +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; + +/** + * The Enum LogFilter. + */ +public enum LogFilter +{ + ALL, + BOT, + NOBOT; + + /** + * Matches. + * + * @param log + * the log + * @return true, if successful + */ + public boolean matches(final HttpAccessLog log) + { + boolean result; + + if (log == null) + { + result = false; + } + else if (this == ALL) + { + result = true; + } + else if ((this == BOT) && (log.isBot())) + { + result = true; + } + else if ((this == NOBOT) && (!log.isBot())) + { + result = true; + } + else + { + result = false; + } + + // + return result; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/core/StatoolInfos.java b/src/fr/devinsy/statoolinfos/core/StatoolInfos.java index 6f9e415..415b9f9 100644 --- a/src/fr/devinsy/statoolinfos/core/StatoolInfos.java +++ b/src/fr/devinsy/statoolinfos/core/StatoolInfos.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -31,8 +32,18 @@ import fr.devinsy.statoolinfos.build.Builder; import fr.devinsy.statoolinfos.crawl.Crawler; import fr.devinsy.statoolinfos.htmlize.Htmlizer; import fr.devinsy.statoolinfos.metrics.Prober; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLogAnalyzer; import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathPropertyUtils; +import fr.devinsy.statoolinfos.stats.ip.IpStat; +import fr.devinsy.statoolinfos.stats.ip.IpStator; +import fr.devinsy.statoolinfos.stats.useragent.UserAgentStat; +import fr.devinsy.statoolinfos.stats.useragent.UserAgentStator; +import fr.devinsy.statoolinfos.stats.visitor.VisitorStat; +import fr.devinsy.statoolinfos.stats.visitor.VisitorStator; +import fr.devinsy.statoolinfos.util.Chrono; +import fr.devinsy.statoolinfos.util.LineIterator; import fr.devinsy.strings.StringList; import fr.devinsy.strings.StringsUtils; @@ -148,6 +159,159 @@ public class StatoolInfos Htmlizer.htmlize(configurationFile); } + /** + * List ips. + * + * @param source + * the source + * @throws IOException + */ + public static void listIps(final File source, final LogFilter filter) + { + try + { + IpStator stator = new IpStator(); + + Chrono chrono = new Chrono().start(); + LineIterator iterator = new LineIterator(source); + while (iterator.hasNext()) + { + String line = iterator.next(); + HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); + + if (filter.matches(log)) + { + stator.putLog(log); + } + } + + for (IpStat stat : stator.getIps()) + { + System.out.println(stat.getValue()); + } + + System.err.println(String.format("%s %10d", "Ip count: ", stator.getIps().size())); + System.err.println(String.format("%s %10d", "Log count: ", stator.getLogCount())); + System.err.println(chrono.format()); + } + catch (IllegalArgumentException exception) + { + System.out.println("Bad format line detected. Aborting…"); + exception.printStackTrace(); + } + catch (DateTimeParseException exception) + { + System.out.println("Bad datetime format detected. Aborting…"); + } + catch (IOException exception) + { + System.out.println("File error detected. Aborting…"); + exception.printStackTrace(); + } + } + + /** + * List user agents. + * + * @param source + * the source + */ + public static void listUserAgents(final File source, final LogFilter filter) + { + try + { + UserAgentStator stator = new UserAgentStator(); + + Chrono chrono = new Chrono().start(); + LineIterator iterator = new LineIterator(source); + while (iterator.hasNext()) + { + String line = iterator.next(); + HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); + + if (filter.matches(log)) + { + stator.putLog(log); + } + } + + for (UserAgentStat stat : stator.getUserAgentStats().sortByLabel()) + { + System.out.println(stat.getValue()); + } + + System.err.println(String.format("%s %10d", "UserAgent count: ", stator.getUserAgentStats().size())); + System.err.println(String.format("%s %10d", "Log count: ", stator.getLogCount())); + System.err.println(chrono.format()); + } + catch (IllegalArgumentException exception) + { + System.out.println("Bad format line detected. Aborting…"); + exception.printStackTrace(); + } + catch (DateTimeParseException exception) + { + System.out.println("Bad datetime format detected. Aborting…"); + exception.printStackTrace(); + } + catch (IOException exception) + { + System.out.println("File error detected. Aborting…"); + exception.printStackTrace(); + } + } + + /** + * List visitors. + * + * @param source + * the source + */ + public static void listVisitors(final File source, final LogFilter filter) + { + try + { + VisitorStator stator = new VisitorStator(); + + Chrono chrono = new Chrono().start(); + LineIterator iterator = new LineIterator(source); + while (iterator.hasNext()) + { + String line = iterator.next(); + HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); + + if (filter.matches(log)) + { + stator.putLog(log); + } + } + + for (VisitorStat stat : stator.getVisitorStats().sortByIp()) + { + System.out.println(stat.getIp() + " " + stat.getUserAgent()); + } + + System.err.println(String.format("%s %10d", "Visitor count: ", stator.getVisitorStats().size())); + System.err.println(String.format("%s %10d", "Log count: ", stator.getLogCount())); + System.err.println(chrono.format()); + } + catch (IllegalArgumentException exception) + { + System.out.println("Bad format line detected. Aborting…"); + exception.printStackTrace(); + } + catch (DateTimeParseException exception) + { + System.out.println("Bad datetime format detected. Aborting…"); + exception.printStackTrace(); + } + catch (IOException exception) + { + System.out.println("File error detected. Aborting…"); + exception.printStackTrace(); + } + } + /** * Stat. * @@ -163,6 +327,157 @@ public class StatoolInfos Prober.probe(configurationFile); } + /** + * Stat ips. + * + * @param source + * the source + */ + public static void statIps(final File source, final LogFilter filter) + { + try + { + IpStator stator = new IpStator(); + + Chrono chrono = new Chrono().start(); + LineIterator iterator = new LineIterator(source); + while (iterator.hasNext()) + { + String line = iterator.next(); + HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); + if (filter.matches(log)) + { + stator.putLog(log); + } + } + + for (IpStat stat : stator.getIps().sortByCount().reverse()) + { + System.out.println(stat.getCount() + " " + stat.getValue()); + } + + System.err.println(String.format("%s %10d", "Ip count: ", stator.getIps().size())); + System.err.println(String.format("%s %10d", "Log count: ", stator.getLogCount())); + System.err.println(chrono.format()); + } + catch (IllegalArgumentException exception) + { + System.out.println("Bad format line detected. Aborting…"); + exception.printStackTrace(); + } + catch (DateTimeParseException exception) + { + System.out.println("Bad datetime format detected. Aborting…"); + exception.printStackTrace(); + } + catch (IOException exception) + { + System.out.println("File error detected. Aborting…"); + exception.printStackTrace(); + } + } + + /** + * Stat user agents. + * + * @param source + * the source + */ + public static void statUserAgents(final File source, final LogFilter filter) + { + try + { + UserAgentStator stator = new UserAgentStator(); + + Chrono chrono = new Chrono().start(); + LineIterator iterator = new LineIterator(source); + while (iterator.hasNext()) + { + String line = iterator.next(); + HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); + if (filter.matches(log)) + { + stator.putLog(log); + } + } + + for (UserAgentStat stat : stator.getUserAgentStats().sortByCount().reverse()) + { + System.out.println(stat.getCount() + " " + stat.getValue()); + } + + System.err.println(String.format("%s %10d", "User Agent count: ", stator.getUserAgentStats().size())); + System.err.println(String.format("%s %10d", "Log count: ", stator.getLogCount())); + System.err.println(chrono.format()); + } + catch (IllegalArgumentException exception) + { + System.out.println("Bad format line detected. Aborting…"); + exception.printStackTrace(); + } + catch (DateTimeParseException exception) + { + System.out.println("Bad datetime format detected. Aborting…"); + exception.printStackTrace(); + } + catch (IOException exception) + { + System.out.println("File error detected. Aborting…"); + exception.printStackTrace(); + } + } + + /** + * Stat visitors. + * + * @param source + * the source + */ + public static void statVisitors(final File source, final LogFilter filter) + { + try + { + VisitorStator stator = new VisitorStator(); + + Chrono chrono = new Chrono().start(); + LineIterator iterator = new LineIterator(source); + while (iterator.hasNext()) + { + String line = iterator.next(); + HttpAccessLog log = HttpAccessLogAnalyzer.parseNginxCombinedLog(line); + if (filter.matches(log)) + { + stator.putLog(log); + } + } + + for (VisitorStat stat : stator.getVisitorStats().sortByVisitCount().reverse()) + { + System.out.println(String.format("%d %d %s %s", stat.getVisits().size(), stat.getLogCount(), stat.getIp(), stat.getUserAgent())); + } + + System.err.println(String.format("%s %10d", "Visitor count: ", stator.getVisitorStats().size())); + System.err.println(String.format("%s %10d", "Log count: ", stator.getLogCount())); + System.err.println(chrono.format()); + } + catch (IllegalArgumentException exception) + { + System.out.println("Bad format line detected. Aborting…"); + exception.printStackTrace(); + } + catch (DateTimeParseException exception) + { + System.out.println("Bad datetime format detected. Aborting…"); + exception.printStackTrace(); + } + catch (IOException exception) + { + System.out.println("File error detected. Aborting…"); + exception.printStackTrace(); + } + + } + /** * Tag date. * @@ -207,5 +522,4 @@ public class StatoolInfos StringsUtils.save(inputFile, lines); } } - } diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java index 281da63..733bc8b 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLog.java @@ -41,7 +41,7 @@ public class HttpAccessLog public static final Pattern IPV4_PATTERN = Pattern.compile("\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}\\.\\d{0,3}"); public static final Pattern IPV6_PATTERN = Pattern.compile("([0-9a-f]{1,4}:{1,2}){4,7}([0-9a-f]){1,4}", Pattern.CASE_INSENSITIVE); - private String remoteAddress; + private String ip; private String remoteUser; private LocalDateTime time; private String request; @@ -55,7 +55,7 @@ public class HttpAccessLog */ public HttpAccessLog() { - this.remoteAddress = null; + this.ip = null; this.remoteUser = null; this.time = null; this.request = null; @@ -80,16 +80,16 @@ public class HttpAccessLog return result; } + public String getIp() + { + return this.ip; + } + public String getReferer() { return this.referer; } - public String getRemoteAddress() - { - return this.remoteAddress; - } - public String getRemoteUser() { return this.remoteUser; @@ -185,7 +185,7 @@ public class HttpAccessLog { boolean result; - if (StringsUtils.containsAnyIgnoreCase(this.userAgent.toString(), "bot", "monitoring", "crawler")) + if (StringsUtils.containsAnyIgnoreCase(this.userAgent.toString(), "bot", "crawler", "monitoring", "HeadlessChrome")) { result = true; } @@ -215,13 +215,13 @@ public class HttpAccessLog { boolean result; - if (this.remoteAddress == null) + if (this.ip == null) { result = false; } else { - result = this.remoteAddress.contains("."); + result = this.ip.contains("."); } // return result; @@ -236,13 +236,13 @@ public class HttpAccessLog { boolean result; - if (this.remoteAddress == null) + if (this.ip == null) { result = false; } else { - result = this.remoteAddress.contains(":"); + result = this.ip.contains(":"); } // return result; @@ -253,16 +253,16 @@ public class HttpAccessLog this.bodyBytesSent = bodyBytesSent; } + public void setIp(final String ip) + { + this.ip = ip; + } + public void setReferer(final String referer) { this.referer = referer; } - public void setRemoteAddress(final String remoteAddress) - { - this.remoteAddress = remoteAddress; - } - public void setRemoteUser(final String remoteUser) { this.remoteUser = remoteUser; @@ -300,7 +300,7 @@ public class HttpAccessLog StringList buffer = new StringList(); - buffer.append("[remoteAddress=").append(this.remoteAddress).append("]"); + buffer.append("[ip=").append(this.ip).append("]"); buffer.append("[remoteUser=").append(this.remoteUser).append("]"); buffer.append("[time=").append(this.time).append("]"); buffer.append("[request=").append(this.request).append("]"); diff --git a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java index b23314d..4709877 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/HttpAccessLogAnalyzer.java @@ -220,28 +220,28 @@ public class HttpAccessLogAnalyzer this.counters.inc("metrics.http.devices." + device.toString().toLowerCase(), year, yearMonth, yearWeek, date); // metrics.http.ip.* = - this.ips.put(log.getRemoteAddress(), year, yearMonth, yearWeek, date); + this.ips.put(log.getIp(), year, yearMonth, yearWeek, date); // metrics.http.ip.ipv4.* = // metrics.http.ip.ipv6.* = if (log.isIPv4()) { - this.ipv4.put(log.getRemoteAddress(), year, yearMonth, yearWeek, date); + this.ipv4.put(log.getIp(), year, yearMonth, yearWeek, date); } else { - this.ipv6.put(log.getRemoteAddress(), year, yearMonth, yearWeek, date); + this.ipv6.put(log.getIp(), year, yearMonth, yearWeek, date); } // metrics.http.ip.bots.* // metrics.http.ip.visitors.* if (log.isBot()) { - this.botIps.put(log.getRemoteAddress(), year, yearMonth, yearWeek, date); + this.botIps.put(log.getIp(), year, yearMonth, yearWeek, date); } else { - this.visitorIps.put(log.getRemoteAddress(), year, yearMonth, yearWeek, date); + this.visitorIps.put(log.getIp(), year, yearMonth, yearWeek, date); } // metrics.http.visits.* = @@ -320,7 +320,7 @@ public class HttpAccessLogAnalyzer if (matcher.matches()) { result = new HttpAccessLog(); - result.setRemoteAddress(matcher.group("remoteAddress")); + result.setIp(matcher.group("remoteAddress")); result.setRemoteUser(matcher.group("remoteUser")); result.setTime(LocalDateTime.parse(matcher.group("time"), DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withLocale(Locale.ENGLISH))); result.setRequest(matcher.group("request")); diff --git a/src/fr/devinsy/statoolinfos/metrics/http/VisitCounters.java b/src/fr/devinsy/statoolinfos/metrics/http/VisitCounters.java index abbee96..823f7e9 100644 --- a/src/fr/devinsy/statoolinfos/metrics/http/VisitCounters.java +++ b/src/fr/devinsy/statoolinfos/metrics/http/VisitCounters.java @@ -57,11 +57,51 @@ public class VisitCounters extends HashMap * the log * @return the string */ - public String computeKey(final HttpAccessLog log) + private String computeKey(final HttpAccessLog log) { String result; - result = String.format("%s--%s", log.getRemoteAddress(), log.getUserAgent().toString()); + result = computeKey(log.getIp(), log.getUserAgent().toString()); + + // + return result; + } + + /** + * Compute key. + * + * @param ip + * the ip + * @param userAgent + * the user agent + * @return the string + */ + private String computeKey(final String ip, final String userAgent) + { + String result; + + result = String.format("%s--%s", ip, userAgent); + + // + return result; + } + + /** + * Gets the. + * + * @param ip + * the ip + * @param userAgent + * the user agent + * @return the visits + */ + public Visits get(final String ip, final String userAgent) + { + Visits result; + + String key = computeKey(ip, userAgent); + + result = get(key); // return result; diff --git a/src/fr/devinsy/statoolinfos/stats/ip/IpStat.java b/src/fr/devinsy/statoolinfos/stats/ip/IpStat.java new file mode 100644 index 0000000..49709e7 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/ip/IpStat.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.ip; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class IpStat. + */ +public final class IpStat +{ + private static Logger logger = LoggerFactory.getLogger(IpStat.class); + + private String value; + private long count; + + /** + * Instantiates a new ip stat. + * + * @param ip + * the ip + */ + public IpStat(final String ip) + { + this.value = ip; + this.count = 0; + } + + public long getCount() + { + return this.count; + } + + public String getValue() + { + return this.value; + } + + public void inc() + { + this.count += 1; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/ip/IpStatComparator.java b/src/fr/devinsy/statoolinfos/stats/ip/IpStatComparator.java new file mode 100644 index 0000000..aa1ba92 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/ip/IpStatComparator.java @@ -0,0 +1,139 @@ +/* + * + */ +package fr.devinsy.statoolinfos.stats.ip; + +import java.util.Comparator; + +import fr.devinsy.statoolinfos.util.CompareUtils; + +/** + * The Class IpStatComparator. + */ +public class IpStatComparator implements Comparator +{ + public enum Sorting + { + IP, + COUNT + } + + private Sorting sorting; + + /** + * Instantiates a new category stat comparator. + * + * @param sorting + * the sorting + */ + public IpStatComparator(final Sorting sorting) + { + this.sorting = sorting; + } + + /** + * Compare. + * + * @param alpha + * the alpha + * @param bravo + * the bravo + * @return the int + */ + @Override + public int compare(final IpStat alpha, final IpStat bravo) + { + int result; + + result = compare(alpha, bravo, this.sorting); + + // + return result; + } + + /** + * Compare. + * + * @param alpha + * the alpha + * @param bravo + * the bravo + * @param sorting + * the sorting + * @return the int + */ + public static int compare(final IpStat alpha, final IpStat bravo, final Sorting sorting) + { + int result; + + if (sorting == null) + { + result = 0; + } + else + { + switch (sorting) + { + default: + case IP: + result = CompareUtils.compare(getIp(alpha), getIp(bravo)); + break; + + case COUNT: + result = CompareUtils.compare(getCount(alpha), getCount(bravo)); + break; + } + } + + // + return result; + } + + /** + * Gets the user count. + * + * @param source + * the source + * @return the user count + */ + public static Long getCount(final IpStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getCount(); + } + + // + return result; + } + + /** + * Gets the name. + * + * @param source + * the source + * @return the name + */ + public static String getIp(final IpStat source) + { + String result; + + if (source == null) + { + result = null; + } + else + { + result = source.getValue(); + } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/stats/ip/IpStatSet.java b/src/fr/devinsy/statoolinfos/stats/ip/IpStatSet.java new file mode 100644 index 0000000..edde0ee --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/ip/IpStatSet.java @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.ip; + +import java.util.HashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class IpStatSet. + */ +public final class IpStatSet extends HashMap +{ + private static final long serialVersionUID = -8411746796941831991L; + + private static Logger logger = LoggerFactory.getLogger(IpStatSet.class); + + /** + * Instantiates a new ip stats. + */ + public IpStatSet() + { + super(); + } + + /** + * Put. + * + * @param ip + * the ip + */ + public void put(final String ip) + { + IpStat stat = get(ip); + if (stat == null) + { + stat = new IpStat(ip); + this.put(ip, stat); + } + + stat.inc(); + } + + /** + * To list. + * + * @return the ip stats + */ + public IpStats toList() + { + IpStats result; + + result = new IpStats(this.size()); + + for (IpStat stat : this.values()) + { + result.add(stat); + } + + // + return result; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/ip/IpStator.java b/src/fr/devinsy/statoolinfos/stats/ip/IpStator.java new file mode 100644 index 0000000..f690200 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/ip/IpStator.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.ip; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; + +/** + * The Class IpStator. + */ +public final class IpStator +{ + private static Logger logger = LoggerFactory.getLogger(IpStator.class); + + private long logCount; + private IpStatSet ips; + + /** + * Instantiates a new user agent stator. + */ + public IpStator() + { + this.logCount = 0; + this.ips = new IpStatSet(); + } + + public IpStats getIps() + { + IpStats result; + + result = this.ips.toList(); + + // + return result; + } + + public long getLogCount() + { + return this.logCount; + } + + /** + * Adds the log. + * + * @param log + * the log + */ + public void putLog(final HttpAccessLog log) + { + this.logCount += 1; + this.ips.put(log.getIp()); + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/ip/IpStats.java b/src/fr/devinsy/statoolinfos/stats/ip/IpStats.java new file mode 100644 index 0000000..0ee2e64 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/ip/IpStats.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.ip; + +import java.util.ArrayList; +import java.util.Collections; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class IpStats. + */ +public final class IpStats extends ArrayList +{ + private static final long serialVersionUID = -2725108375443481335L; + private static Logger logger = LoggerFactory.getLogger(IpStats.class); + + /** + * Instantiates a new ip user agent stats. + */ + public IpStats() + { + super(); + } + + /** + * Instantiates a new ip stats. + * + * @param capacity + * the capacity + */ + public IpStats(final int capacity) + { + super(capacity); + } + + /** + * Reverse. + * + * @return the ip stats + */ + public IpStats reverse() + { + IpStats result; + + Collections.reverse(this); + + result = this; + + // + return result; + } + + /** + * Sort. + * + * @param sorting + * the sorting + * @return the ip stats + */ + public IpStats sort(final IpStatComparator.Sorting sorting) + { + IpStats result; + + sort(new IpStatComparator(sorting)); + + result = this; + + // + return result; + } + + /** + * Sort count. + * + * @return the ip stats + */ + public IpStats sortByCount() + { + IpStats result; + + result = sort(IpStatComparator.Sorting.COUNT); + + // + return result; + } + + /** + * Sort by ip. + * + * @return the ip stats + */ + public IpStats sortByIp() + { + IpStats result; + + result = sort(IpStatComparator.Sorting.IP); + + // + return result; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStat.java b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStat.java new file mode 100644 index 0000000..0b25b36 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStat.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.useragent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class UserAgentStat. + */ +public final class UserAgentStat +{ + private static Logger logger = LoggerFactory.getLogger(UserAgentStat.class); + + private String value; + private long count; + private long ipLinkCount; + + /** + * Instantiates a new user agent stat. + * + * @param ip + * the ip + * @param userAgent + * the user agent + */ + public UserAgentStat(final String userAgent) + { + this.value = userAgent; + this.count = 0; + this.ipLinkCount = 0; + } + + public long getCount() + { + return this.count; + } + + public long getIpLinkCount() + { + return this.ipLinkCount; + } + + public String getValue() + { + return this.value; + } + + public void inc() + { + this.count += 1; + } + + public void setIpLinkCount(final long ipLinkCount) + { + this.ipLinkCount = ipLinkCount; + } + +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStatComparator.java b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStatComparator.java new file mode 100644 index 0000000..5cfe770 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStatComparator.java @@ -0,0 +1,139 @@ +/* + * + */ +package fr.devinsy.statoolinfos.stats.useragent; + +import java.util.Comparator; + +import fr.devinsy.statoolinfos.util.CompareUtils; + +/** + * The Class UserAgentStatComparator. + */ +public class UserAgentStatComparator implements Comparator +{ + public enum Sorting + { + USERAGENT, + COUNT + } + + private Sorting sorting; + + /** + * Instantiates a new category stat comparator. + * + * @param sorting + * the sorting + */ + public UserAgentStatComparator(final Sorting sorting) + { + this.sorting = sorting; + } + + /** + * Compare. + * + * @param alpha + * the alpha + * @param bravo + * the bravo + * @return the int + */ + @Override + public int compare(final UserAgentStat alpha, final UserAgentStat bravo) + { + int result; + + result = compare(alpha, bravo, this.sorting); + + // + return result; + } + + /** + * Compare. + * + * @param alpha + * the alpha + * @param bravo + * the bravo + * @param sorting + * the sorting + * @return the int + */ + public static int compare(final UserAgentStat alpha, final UserAgentStat bravo, final Sorting sorting) + { + int result; + + if (sorting == null) + { + result = 0; + } + else + { + switch (sorting) + { + default: + case USERAGENT: + result = CompareUtils.compare(getUserAgent(alpha), getUserAgent(bravo)); + break; + + case COUNT: + result = CompareUtils.compare(getCount(alpha), getCount(bravo)); + break; + } + } + + // + return result; + } + + /** + * Gets the count. + * + * @param source + * the source + * @return the count + */ + public static Long getCount(final UserAgentStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getCount(); + } + + // + return result; + } + + /** + * Gets the user agent. + * + * @param source + * the source + * @return the user agent + */ + public static String getUserAgent(final UserAgentStat source) + { + String result; + + if (source == null) + { + result = null; + } + else + { + result = source.getValue(); + } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStatSet.java b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStatSet.java new file mode 100644 index 0000000..22e3fff --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStatSet.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.useragent; + +import java.util.HashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class UserAgents. + */ +public final class UserAgentStatSet extends HashMap +{ + private static final long serialVersionUID = -7943808966632477322L; + + private static Logger logger = LoggerFactory.getLogger(UserAgentStatSet.class); + + /** + * Instantiates a new user agents. + */ + public UserAgentStatSet() + { + super(); + } + + public void put(final String userAgent) + { + UserAgentStat stat = get(userAgent); + if (stat == null) + { + stat = new UserAgentStat(userAgent); + this.put(userAgent, stat); + } + + stat.inc(); + } + + /** + * Put. + * + * @param userAgent + * the user agent + */ + public void put(final UserAgentStat userAgent) + { + this.put(userAgent.getValue(), userAgent); + } + + /** + * To list. + * + * @return the user agent stats + */ + public UserAgentStats toList() + { + UserAgentStats result; + + result = new UserAgentStats(this.size()); + + for (UserAgentStat stat : this.values()) + { + result.add(stat); + } + + // + return result; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStator.java b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStator.java new file mode 100644 index 0000000..da9879e --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStator.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.useragent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; + +/** + * The Class UserAgentStator. + */ +public final class UserAgentStator +{ + private static Logger logger = LoggerFactory.getLogger(UserAgentStator.class); + + private long logCount; + private UserAgentStatSet userAgents; + + /** + * Instantiates a new user agent stator. + */ + public UserAgentStator() + { + this.logCount = 0; + this.userAgents = new UserAgentStatSet(); + } + + public long getLogCount() + { + return this.logCount; + } + + /** + * Gets the user agent stats. + * + * @return the user agent stats + */ + public UserAgentStats getUserAgentStats() + { + UserAgentStats result; + + result = this.userAgents.toList(); + + // + return result; + } + + /** + * Adds the log. + * + * @param log + * the log + */ + public void putLog(final HttpAccessLog log) + { + String userAgent = log.getUserAgent().toString().trim(); + + this.logCount += 1; + this.userAgents.put(userAgent); + } +} diff --git a/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStats.java b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStats.java new file mode 100644 index 0000000..6d3967f --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/useragent/UserAgentStats.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.useragent; + +import java.util.ArrayList; +import java.util.Collections; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class UserAgentStats. + */ +public final class UserAgentStats extends ArrayList +{ + private static final long serialVersionUID = -6389473662022565639L; + private static Logger logger = LoggerFactory.getLogger(UserAgentStats.class); + + /** + * Instantiates a new user agent stats. + */ + public UserAgentStats() + { + super(); + } + + /** + * Instantiates a new user agent stats. + * + * @param capacity + * the capacity + */ + public UserAgentStats(final int capacity) + { + super(capacity); + } + + /** + * Reverse. + * + * @return the user agent stats + */ + public UserAgentStats reverse() + { + UserAgentStats result; + + Collections.reverse(this); + + result = this; + + // + return result; + } + + /** + * Sort. + * + * @param sorting + * the sorting + * @return the ip stats + */ + public UserAgentStats sort(final UserAgentStatComparator.Sorting sorting) + { + UserAgentStats result; + + sort(new UserAgentStatComparator(sorting)); + + result = this; + + // + return result; + } + + /** + * Sort by count. + * + * @return the user agent stats + */ + public UserAgentStats sortByCount() + { + UserAgentStats result; + + result = sort(UserAgentStatComparator.Sorting.COUNT); + + // + return result; + } + + /** + * Sort by ip. + * + * @return the user agent stats + */ + public UserAgentStats sortByLabel() + { + UserAgentStats result; + + result = sort(UserAgentStatComparator.Sorting.USERAGENT); + + // + return result; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStat.java b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStat.java new file mode 100644 index 0000000..3deba26 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStat.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.visitor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.metrics.http.Visits; + +/** + * The Class VisitorStat. + */ +public final class VisitorStat +{ + private static Logger logger = LoggerFactory.getLogger(VisitorStat.class); + + private String ip; + private String userAgent; + private long logCount; + private Visits visits; + + /** + * Instantiates a new visitor stat. + * + * @param ip + * the ip + * @param userAgent + * the user agent + */ + public VisitorStat(final String ip, final String userAgent) + { + this.ip = ip; + this.userAgent = userAgent; + this.logCount = 0; + this.visits = new Visits(); + } + + public String getIp() + { + return this.ip; + } + + public long getLogCount() + { + return this.logCount; + } + + public String getUserAgent() + { + return this.userAgent; + } + + /** + * Gets the visit count. + * + * @return the visit count + */ + public long getVisitCount() + { + long result; + + result = this.visits.size(); + + // + return result; + } + + public Visits getVisits() + { + return this.visits; + } + + public void inc() + { + this.logCount += 1; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStatComparator.java b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStatComparator.java new file mode 100644 index 0000000..03fddaa --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStatComparator.java @@ -0,0 +1,190 @@ +/* + * + */ +package fr.devinsy.statoolinfos.stats.visitor; + +import java.util.Comparator; + +import fr.devinsy.statoolinfos.util.CompareUtils; + +/** + * The Class VisitorStatComparator. + */ +public class VisitorStatComparator implements Comparator +{ + public enum Sorting + { + IP, + USERAGENT, + LOGCOUNT, + VISITCOUNT + } + + private Sorting sorting; + + /** + * Instantiates a new visitor stat comparator. + * + * @param sorting + * the sorting + */ + public VisitorStatComparator(final Sorting sorting) + { + this.sorting = sorting; + } + + /** + * Compare. + * + * @param alpha + * the alpha + * @param bravo + * the bravo + * @return the int + */ + @Override + public int compare(final VisitorStat alpha, final VisitorStat bravo) + { + int result; + + result = compare(alpha, bravo, this.sorting); + + // + return result; + } + + /** + * Compare. + * + * @param alpha + * the alpha + * @param bravo + * the bravo + * @param sorting + * the sorting + * @return the int + */ + public static int compare(final VisitorStat alpha, final VisitorStat bravo, final Sorting sorting) + { + int result; + + if (sorting == null) + { + result = 0; + } + else + { + switch (sorting) + { + case IP: + result = CompareUtils.compare(getIp(alpha), getIp(bravo)); + break; + + default: + case USERAGENT: + result = CompareUtils.compare(getUserAgent(alpha), getUserAgent(bravo)); + break; + + case LOGCOUNT: + result = CompareUtils.compare(getLogCount(alpha), getLogCount(bravo)); + break; + + case VISITCOUNT: + result = CompareUtils.compare(getVisitCount(alpha), getVisitCount(bravo)); + break; + } + } + + // + return result; + } + + /** + * Gets the ip. + * + * @param source + * the source + * @return the ip + */ + public static String getIp(final VisitorStat source) + { + String result; + + if (source == null) + { + result = null; + } + else + { + result = source.getIp(); + } + + // + return result; + } + + /** + * Gets the count. + * + * @param source + * the source + * @return the count + */ + public static Long getLogCount(final VisitorStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getLogCount(); + } + + // + return result; + } + + /** + * Gets the user agent. + * + * @param source + * the source + * @return the user agent + */ + public static String getUserAgent(final VisitorStat source) + { + String result; + + if (source == null) + { + result = null; + } + else + { + result = source.getUserAgent(); + } + + // + return result; + } + + public static Long getVisitCount(final VisitorStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getVisitCount(); + } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStatSet.java b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStatSet.java new file mode 100644 index 0000000..9d5bd16 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStatSet.java @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.visitor; + +import java.util.HashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.stats.useragent.UserAgentStat; + +/** + * The Class VisitorStatSet. + */ +public final class VisitorStatSet extends HashMap +{ + private static final long serialVersionUID = -3123113539922160408L; + private static Logger logger = LoggerFactory.getLogger(VisitorStatSet.class); + + /** + * Instantiates a new visitor stat set. + */ + public VisitorStatSet() + { + super(); + } + + /** + * Compute key. + * + * @param ip + * the ip + * @param userAgent + * the user agent + * @return the string + */ + private String computeKey(final String ip, final String userAgent) + { + String result; + + result = ip + userAgent.trim(); + + // + return result; + } + + /** + * Put. + * + * @param userAgent + * the user agent + */ + public VisitorStat put(final String ip, final String userAgent) + { + VisitorStat result; + + String key = computeKey(ip, userAgent); + + result = get(key); + if (result == null) + { + result = new VisitorStat(ip, userAgent); + this.put(key, result); + } + + result.inc(); + + // + return result; + } + + /** + * Put. + * + * @param userAgent + * the user agent + */ + public void put(final String ip, final UserAgentStat userAgent) + { + this.put(userAgent.getValue(), userAgent); + } + + /** + * To list. + * + * @return the user agent stats + */ + public VisitorStats toList() + { + VisitorStats result; + + result = new VisitorStats(this.size()); + + for (VisitorStat stat : this.values()) + { + result.add(stat); + } + + // + return result; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStator.java b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStator.java new file mode 100644 index 0000000..6a29e5f --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStator.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.visitor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.metrics.http.HttpAccessLog; +import fr.devinsy.statoolinfos.metrics.http.HttpStatusCategory; + +/** + * The Class VisitorStator. + */ +public final class VisitorStator +{ + private static Logger logger = LoggerFactory.getLogger(VisitorStator.class); + + private long logCount; + private VisitorStatSet visitors; + + /** + * Instantiates a new visitor stator. + */ + public VisitorStator() + { + this.logCount = 0; + this.visitors = new VisitorStatSet(); + } + + public long getLogCount() + { + return this.logCount; + } + + /** + * Gets the visitors. + * + * @return the visitors + */ + public VisitorStats getVisitorStats() + { + VisitorStats result; + + result = this.visitors.toList(); + + // + return result; + } + + /** + * Adds the log. + * + * @param log + * the log + */ + public void putLog(final HttpAccessLog log) + { + if (log != null) + { + this.logCount += 1; + VisitorStat stat = this.visitors.put(log.getIp(), log.getUserAgent().toString()); + + if (log.getStatus().getCategory() == HttpStatusCategory.SUCCESS) + { + stat.getVisits().add(log.getTime()); + } + } + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStats.java b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStats.java new file mode 100644 index 0000000..311370d --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/visitor/VisitorStats.java @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.visitor; + +import java.util.ArrayList; +import java.util.Collections; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The Class VisitorStats. + */ +public final class VisitorStats extends ArrayList +{ + private static final long serialVersionUID = 8866450468875949301L; + private static Logger logger = LoggerFactory.getLogger(VisitorStats.class); + + /** + * Instantiates a new visitor stats. + */ + public VisitorStats() + { + super(); + } + + /** + * Instantiates a new visitor stats. + * + * @param capacity + * the capacity + */ + public VisitorStats(final int capacity) + { + super(capacity); + } + + /** + * Reverse. + * + * @return the visitor stats + */ + public VisitorStats reverse() + { + VisitorStats result; + + Collections.reverse(this); + + result = this; + + // + return result; + } + + /** + * Sort. + * + * @param sorting + * the sorting + * @return the visitor stats + */ + public VisitorStats sort(final VisitorStatComparator.Sorting sorting) + { + VisitorStats result; + + sort(new VisitorStatComparator(sorting)); + + result = this; + + // + return result; + } + + /** + * Sort by ip. + * + * @return the visitor stats + */ + public VisitorStats sortByIp() + { + VisitorStats result; + + result = sort(VisitorStatComparator.Sorting.IP); + + // + return result; + } + + /** + * Sort by count. + * + * @return the visitor stats + */ + public VisitorStats sortByLogCount() + { + VisitorStats result; + + result = sort(VisitorStatComparator.Sorting.LOGCOUNT); + + // + return result; + } + + /** + * Sort by label. + * + * @return the visitor stats + */ + public VisitorStats sortByUserAgent() + { + VisitorStats result; + + result = sort(VisitorStatComparator.Sorting.USERAGENT); + + // + return result; + } + + /** + * Sort by visit count. + * + * @return the visitor stats + */ + public VisitorStats sortByVisitCount() + { + VisitorStats result; + + result = sort(VisitorStatComparator.Sorting.VISITCOUNT); + + // + return result; + } +} \ No newline at end of file diff --git a/src/fr/devinsy/statoolinfos/util/Chrono.java b/src/fr/devinsy/statoolinfos/util/Chrono.java index c3debba..38d4321 100644 --- a/src/fr/devinsy/statoolinfos/util/Chrono.java +++ b/src/fr/devinsy/statoolinfos/util/Chrono.java @@ -27,7 +27,11 @@ import java.time.ZoneOffset; */ public class Chrono { + public static final long DEFAULT_MESSAGE_INTERVAL = 60; + private LocalDateTime start; + private long previousDurationMessage; + private long messageInterval; /** * Instantiates a new time keeper. @@ -35,6 +39,27 @@ public class Chrono public Chrono() { reset(); + this.messageInterval = DEFAULT_MESSAGE_INTERVAL; + } + + /** + * Check message. + * + * @param current + * the current + * @param max + * the max + */ + public void checkMessage(final long current, final long max) + { + long currentDuration = duration(); + + if ((currentDuration % this.messageInterval == 0) && (currentDuration != this.previousDurationMessage)) + { + this.previousDurationMessage = currentDuration; + System.out.println(format()); + System.out.println(String.format("%s %d/%d", format(), current, max)); + } } /** @@ -83,6 +108,18 @@ public class Chrono public void reset() { this.start = null; + this.previousDurationMessage = 0; + } + + /** + * Sets the check intervale. + * + * @param seconds + * the new check intervale + */ + public void setMessageInterval(final long seconds) + { + this.messageInterval = seconds; } /** @@ -93,6 +130,7 @@ public class Chrono Chrono result; this.start = LocalDateTime.now(); + this.previousDurationMessage = 0L; result = this; diff --git a/test/fr/devinsy/statoolinfos/metrics/http/VisitCountersTest.java b/test/fr/devinsy/statoolinfos/metrics/http/VisitCountersTest.java index 9c78ebf..07832b6 100644 --- a/test/fr/devinsy/statoolinfos/metrics/http/VisitCountersTest.java +++ b/test/fr/devinsy/statoolinfos/metrics/http/VisitCountersTest.java @@ -52,7 +52,7 @@ public class VisitCountersTest // HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setTime(LocalDateTime.of(2020, 10, 20, 23, 0)); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); visitCounters.putVisit(log); @@ -84,14 +84,14 @@ public class VisitCountersTest // { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0)); visitCounters.putVisit(log); } { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20)); visitCounters.putVisit(log); @@ -124,21 +124,21 @@ public class VisitCountersTest // { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0)); visitCounters.putVisit(log); } { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20)); visitCounters.putVisit(log); } { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 55)); visitCounters.putVisit(log); @@ -171,21 +171,21 @@ public class VisitCountersTest // { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0)); visitCounters.putVisit(log); } { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 55)); visitCounters.putVisit(log); } { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20)); visitCounters.putVisit(log); @@ -218,21 +218,21 @@ public class VisitCountersTest // { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 0)); visitCounters.putVisit(log); } { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 35)); visitCounters.putVisit(log); } { HttpAccessLog log = new HttpAccessLog(); - log.setRemoteAddress("192.168.1.1"); + log.setIp("192.168.1.1"); log.setUserAgent(new UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36")); log.setTime(LocalDateTime.of(2020, 10, 20, 12, 20)); visitCounters.putVisit(log);