From c2076f15cebbe8fd15aed6a709f1154713419f72 Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Tue, 29 Mar 2022 01:47:17 +0200 Subject: [PATCH] Extended Etherpad probing. --- README.md | 16 + .../devinsy/statoolinfos/metrics/Prober.java | 6 +- .../etherpad/EtherpadHttpLogAnalyzer.java | 3 +- .../metrics/etherpad/EtherpadLog.java | 291 ++++++++++++++++++ .../metrics/etherpad/EtherpadLogProber.java | 224 ++++++++++++++ .../metrics/etherpad/EtherpadProber.java | 15 +- 6 files changed, 545 insertions(+), 10 deletions(-) create mode 100644 src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLog.java create mode 100644 src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLogProber.java diff --git a/README.md b/README.md index 20c2d44..8bf583e 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,23 @@ Create a cron file to update the metric file everyday: Warning: in previous day mode, the metrics generated are overwrited for the last month, the last week and the last day. So, six weeks in logs are required. +### Etherpad metrics +Configuration template: + +``` +conf.probe.types=Etherpad +conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log* +conf.probe.httpaccesslog.pattern= +# jdbc:mariadb://localhost:1234/databasename +# jdbc:mysql://localhost:1234/databasename +# jdbc:postgresql://localhost:1234/databasename +# jdbc:sqlite:/foo/bar/databasename.sqlite +conf.probe.gitea.database.url= +conf.probe.gitea.database.user= +conf.probe.gitea.database.password= +conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties +``` ### Framdadate metrics (coming soon) diff --git a/src/fr/devinsy/statoolinfos/metrics/Prober.java b/src/fr/devinsy/statoolinfos/metrics/Prober.java index c2c2fe3..b6b4a5e 100644 --- a/src/fr/devinsy/statoolinfos/metrics/Prober.java +++ b/src/fr/devinsy/statoolinfos/metrics/Prober.java @@ -140,12 +140,14 @@ public class Prober logger.info("== Probing Etherpad."); String httpLogs = configuration.getProbeHttpAccessLogSource(); logger.info("httpLogs=[{}]", httpLogs); + String logs = configuration.get("conf.probe.etherpad.logs"); + logger.info("logs=[{}]", logs); String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern(); logger.info("httpAccessPattern=[{}]", httpAccessLogRegex); - DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.gitea"); + DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.etherpad"); logger.info("database={}", database.toString()); - PathCounters metrics = EtherpadProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, database); + PathCounters metrics = EtherpadProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, FilesUtils.searchByWildcard(logs), database); counters.putAll(metrics); } diff --git a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java index cef39f3..3a96250 100644 --- a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java +++ b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadHttpLogAnalyzer.java @@ -38,8 +38,7 @@ public class EtherpadHttpLogAnalyzer { private static Logger logger = LoggerFactory.getLogger(EtherpadHttpLogAnalyzer.class); - public static final Pattern USE_PATTERN = Pattern.compile("GET /p/\\S+ "); - public static final Pattern CREATE_PATTERN = Pattern.compile("POST / .*"); + public static final Pattern USE_PATTERN = Pattern.compile("^GET /p/\\S+ .*$"); private PathCounters counters; private UserCounters users; diff --git a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLog.java b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLog.java new file mode 100644 index 0000000..eabd7e6 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLog.java @@ -0,0 +1,291 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.etherpad; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.metrics.TimeMark; +import fr.devinsy.statoolinfos.metrics.TimeMarkUtils; +import fr.devinsy.strings.StringList; + +/** + * The Class EtherpadLog. + */ +public class EtherpadLog +{ + private static Logger logger = LoggerFactory.getLogger(EtherpadLog.class); + + private LocalDateTime time; + private String level; + private String type; + private String event; + private String padname; + private String ip; + private String author; + + /** + * Instantiates a new etherpad log. + */ + public EtherpadLog() + { + this.time = null; + this.level = null; + this.type = null; + this.event = null; + this.padname = null; + this.ip = null; + this.author = null; + } + + public String getAuthor() + { + return this.author; + } + + /** + * Gets the date. + * + * @return the date + */ + public String getDate() + { + String result; + + result = this.time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE)); + + // + return result; + } + + public String getEvent() + { + return this.event; + } + + public String getIp() + { + return this.ip; + } + + public String getLevel() + { + return this.level; + } + + public String getPadname() + { + return this.padname; + } + + public LocalDateTime getTime() + { + return this.time; + } + + /** + * Gets the time marks. + * + * @return the time marks + */ + public StringList getTimeMarks() + { + StringList result; + + result = new StringList(); + + String year = TimeMark.yearOf(this.time).toString(); + String yearMonth = TimeMark.yearMonthOf(this.time).toString(); + String yearWeek = TimeMark.yearWeekOf(this.time).toString(); + String date = TimeMark.dayOf(this.time).toString(); + + result.add(year); + result.add(yearMonth); + result.add(yearWeek); + result.add(date); + + // + return result; + } + + public String getType() + { + return this.type; + } + + /** + * Gets the year. + * + * @return the year + */ + public String getYear() + { + String result; + + if (this.time == null) + { + result = null; + } + else + { + result = TimeMarkUtils.yearOf(this.time); + } + + // + return result; + } + + /** + * Gets the year month. + * + * @return the year month + */ + public String getYearMonth() + { + String result; + + if (this.time == null) + { + result = null; + } + else + { + result = TimeMarkUtils.yearMonthOf(this.time); + } + // + return result; + } + + /** + * Gets the year week. + * + * @return the year week + */ + public String getYearWeek() + { + String result; + if (this.time == null) + { + result = null; + } + else + { + result = TimeMarkUtils.yearWeekOf(this.time); + } + // + return result; + } + + /** + * Checks if is ipv4. + * + * @return true, if is ipv4 + */ + public boolean isIPv4() + { + boolean result; + + if (this.ip == null) + { + result = false; + } + else + { + result = this.ip.contains("."); + } + + // + return result; + } + + /** + * Checks if is ipv6. + * + * @return true, if is ipv6 + */ + public boolean isIPv6() + { + boolean result; + + if (this.ip == null) + { + result = false; + } + else + { + result = this.ip.contains(":"); + } + + // + return result; + } + + public void setAuthor(final String author) + { + this.author = author; + } + + public void setEvent(final String event) + { + this.event = event; + } + + public void setIp(final String ip) + { + this.ip = ip; + } + + public void setLevel(final String level) + { + this.level = level; + } + + public void setPadname(final String padname) + { + this.padname = padname; + } + + public void setTime(final LocalDateTime time) + { + this.time = time; + } + + public void setType(final String type) + { + this.type = type; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() + { + String result; + + result = String.format("[time=%s, level=%s, type=%s, event=%s, %padname=%s, ip=%s, auhtor=%s]", this.time, this.level, this.type, this.event, this.padname, this.ip, this.author); + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLogProber.java b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLogProber.java new file mode 100644 index 0000000..c5654c9 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLogProber.java @@ -0,0 +1,224 @@ +/* + * Copyright (C) 2022 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.metrics.etherpad; + +import java.io.File; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.metrics.PathCounters; +import fr.devinsy.statoolinfos.metrics.UserCounters; +import fr.devinsy.statoolinfos.util.Files; +import fr.devinsy.statoolinfos.util.LineIterator; + +/** + * The Class EtherpadLogProber. + */ +public class EtherpadLogProber +{ + private static Logger logger = LoggerFactory.getLogger(EtherpadLogProber.class); + + // 2021-11-17 01:51:59 DELETE 7922dd508e0cf9b2 + public static final Pattern LOG_PATTERN = Pattern.compile( + "^\\[(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(\\.\\d{3})?)]\\s\\[(?[^]]+)]\\s+(?\\S+) - \\[(?[^]]+)]\\spad:(?\\S+)\\ssocket:\\S+\\sIP:(?\\S+) authorID:(?\\S+)( username:.+)?$"); + /* + [2022-03-28 02:19:35.842] [INFO] access - [ENTER] pad:test socket:Qr8BEX3mP6EmheyLAAAA IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin + [2022-03-28 18:50:10.963] [INFO] access - [CREATE] pad:test7 socket:WddndPw_4vt01-0YAAAC IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin + [2022-03-28 18:52:00.668] [INFO] access - [ENTER] pad:test7 socket:gA4Ba57988zrGJNmAAAD IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin + [2022-03-28 18:52:27.947] [INFO] access - [ENTER] pad:test7 socket:9ZrFheETmDUpUcS1AAAE IP:82.65.227.202 authorID:a.AMpqExl3JHbPL52c + [2022-03-28 18:59:58.501] [INFO] access - [ENTER] pad:test socket:mCgPMxg2wb7upHP8AAAG IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin + [2022-03-28 19:01:24.884] [INFO] access - [LEAVE] pad:test socket:mCgPMxg2wb7upHP8AAAG IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin + */ + + public static final Pattern DELETE_EMPTY_LOG_PATTERN = Pattern.compile( + "^\\[(?\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(\\.\\d{3})?)]\\s\\[(?[^]]+)]\\s+(?\\S+) - Deleting (?\\S+) when user leaved since empty$"); + /* + [2022-03-29 00:43:57.085] [INFO] ep_delete_empty_pads - Deleting test8 when user leaved since empty + */ + + /** + * Instantiates a new etherpad log prober. + */ + private EtherpadLogProber() + { + } + + /** + * Parses the log. + * + * @param line + * the line + * @return the http log + */ + public static EtherpadLog parseLog(final String line) + { + EtherpadLog result; + + Matcher matcher = LOG_PATTERN.matcher(line); + if (matcher.matches()) + { + result = new EtherpadLog(); + result.setTime(LocalDateTime.parse(matcher.group("datetime"), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withLocale(Locale.ENGLISH))); + result.setLevel(matcher.group("level")); + result.setType(matcher.group("type")); + result.setEvent(matcher.group("event")); + result.setPadname(matcher.group("padname")); + result.setIp(matcher.group("ip")); + result.setAuthor(matcher.group("author")); + } + else + { + matcher = DELETE_EMPTY_LOG_PATTERN.matcher(line); + if (matcher.matches()) + { + result = new EtherpadLog(); + result.setTime(LocalDateTime.parse(matcher.group("datetime"), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withLocale(Locale.ENGLISH))); + result.setLevel(matcher.group("level")); + result.setType(matcher.group("type")); + result.setEvent("Deleting"); + result.setAuthor(matcher.group("padname")); + } + else + { + result = null; + } + } + + // + return result; + } + + /** + * Probe. + * + * @param file + * the file + * @return the path counters + */ + public static PathCounters probe(final Files files) throws IOException + { + PathCounters result; + + result = new PathCounters(); + + UserCounters users = new UserCounters(); + UserCounters ipv4Users = new UserCounters(); + UserCounters ipv6Users = new UserCounters(); + UserCounters actives = new UserCounters(); + + for (File file : files) + { + System.out.println("Probing file [" + file.getAbsolutePath() + "]"); + + // + int lineCount = 0; + LineIterator iterator = new LineIterator(file); + while (iterator.hasNext()) + { + lineCount += 1; + String line = iterator.next(); + + // Remove ANSI Escape Codes. + line = line.replaceAll("\u001B\\[[\\d;]*[^\\d;]", ""); + + try + { + EtherpadLog log = parseLog(line); + // System.out.println("====> " + log); + if (log == null) + { + if (line.contains("access -")) + { + logger.warn("LINE IS NOT MATCHING [{}]", line); + } + } + else + { + // Timemarks. + String year = log.getYear(); + String yearMonth = log.getYearMonth(); + String yearWeek = log.getYearWeek(); + String date = log.getDate(); + + if (StringUtils.equals(log.getType(), "access")) + { + // metrics.service.users + // metrics.service.users.ipv4 + // metrics.service.users.ipv6 + users.put(log.getAuthor(), year, yearMonth, yearWeek, date); + if (log.isIPv4()) + { + ipv4Users.put(log.getAuthor(), year, yearMonth, yearWeek, date); + } + else + { + ipv6Users.put(log.getAuthor(), year, yearMonth, yearWeek, date); + } + + // metrics.etherpad.pads.actives + actives.put(log.getPadname(), year, yearMonth, yearWeek, date); + + // metrics.etherpad.pads.created + // alias metrics.textprocessors.created + if (StringUtils.equals(log.getEvent(), "CREATE")) + { + result.inc("metrics.etherpad.pads.created", year, yearMonth, yearWeek, date); + result.inc("metrics.textprocessors.created", year, yearMonth, yearWeek, date); + } + + if (StringUtils.equalsAny(log.getEvent(), "CREATE", "ENTER")) + { + result.inc("metrics.etherpad.pads.reads", year, yearMonth, yearWeek, date); + result.inc("metrics.textprocessors.reads", year, yearMonth, yearWeek, date); + } + } + else if (StringUtils.equals(log.getType(), "ep_delete_empty_pads")) + { + result.inc("metrics.etherpad.pads.deleted", year, yearMonth, yearWeek, date); + result.inc("metrics.textprocessors.deleted", year, yearMonth, yearWeek, date); + } + } + } + catch (Exception exception) + { + logger.warn("Error parsing line [{}][{}]", line, exception.getMessage()); + exception.printStackTrace(); + } + } + System.out.println("PrivatebinPatchLog line count=" + lineCount); + } + + // + result.putAll(users.getCounters("metrics.service.users")); + result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4")); + result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6")); + result.putAll(actives.getCounters("metrics.etherpad.pads.actives")); + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java index edc7d6e..9828d31 100644 --- a/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java +++ b/src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadProber.java @@ -61,14 +61,21 @@ public class EtherpadProber * @throws StatoolInfosException * the statool infos exception */ - public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException + public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final Files logs, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException { PathCounters result; // metrics.service.users // metrics.service.users.ipv4 // metrics.service.users.ipv6 - result = EtherpadHttpLogAnalyzer.probe(httpLogs, httpLogRegex); + if (logs.isEmpty()) + { + result = EtherpadHttpLogAnalyzer.probe(httpLogs, httpLogRegex); + } + else + { + result = EtherpadLogProber.probe(logs); + } try { @@ -144,10 +151,6 @@ public class EtherpadProber logger.error("ERROR with database.", exception); } - /* - * select substring(key, strpos(key, ':')+1, strpos(substring(key, strpos(key, ':')+1), ':')) from store where key like 'pad:%:revs:%'; - */ - // return result; }