Extended Etherpad probing.
This commit is contained in:
parent
5601fe5d54
commit
c2076f15ce
6 changed files with 545 additions and 10 deletions
16
README.md
16
README.md
|
|
@ -126,7 +126,23 @@ Create a cron file to update the metric file everyday:
|
|||
|
||||
Warning: in previous day mode, the metrics generated are overwrited for the last month, the last week and the last day. So, six weeks in logs are required.
|
||||
|
||||
### Etherpad metrics
|
||||
|
||||
Configuration template:
|
||||
|
||||
```
|
||||
conf.probe.types=Etherpad
|
||||
conf.probe.httpaccesslog.file=/var/log/apache2/foo.bar.org-access.log*
|
||||
conf.probe.httpaccesslog.pattern=
|
||||
# jdbc:mariadb://localhost:1234/databasename
|
||||
# jdbc:mysql://localhost:1234/databasename
|
||||
# jdbc:postgresql://localhost:1234/databasename
|
||||
# jdbc:sqlite:/foo/bar/databasename.sqlite
|
||||
conf.probe.gitea.database.url=
|
||||
conf.probe.gitea.database.user=
|
||||
conf.probe.gitea.database.password=
|
||||
conf.probe.target=/srv/statoolinfos/well-known/statoolinfos/foo.bar.org-metrics.properties
|
||||
```
|
||||
|
||||
### Framdadate metrics (coming soon)
|
||||
|
||||
|
|
|
|||
|
|
@ -140,12 +140,14 @@ public class Prober
|
|||
logger.info("== Probing Etherpad.");
|
||||
String httpLogs = configuration.getProbeHttpAccessLogSource();
|
||||
logger.info("httpLogs=[{}]", httpLogs);
|
||||
String logs = configuration.get("conf.probe.etherpad.logs");
|
||||
logger.info("logs=[{}]", logs);
|
||||
String httpAccessLogRegex = configuration.getProbeHttpAccessLogPattern();
|
||||
logger.info("httpAccessPattern=[{}]", httpAccessLogRegex);
|
||||
DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.gitea");
|
||||
DatabaseConfig database = configuration.getDatabaseConfig("conf.probe.etherpad");
|
||||
logger.info("database={}", database.toString());
|
||||
|
||||
PathCounters metrics = EtherpadProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, database);
|
||||
PathCounters metrics = EtherpadProber.probe(FilesUtils.searchByWildcard(httpLogs), httpAccessLogRegex, FilesUtils.searchByWildcard(logs), database);
|
||||
counters.putAll(metrics);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -38,8 +38,7 @@ public class EtherpadHttpLogAnalyzer
|
|||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(EtherpadHttpLogAnalyzer.class);
|
||||
|
||||
public static final Pattern USE_PATTERN = Pattern.compile("GET /p/\\S+ ");
|
||||
public static final Pattern CREATE_PATTERN = Pattern.compile("POST / .*");
|
||||
public static final Pattern USE_PATTERN = Pattern.compile("^GET /p/\\S+ .*$");
|
||||
|
||||
private PathCounters counters;
|
||||
private UserCounters users;
|
||||
|
|
|
|||
291
src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLog.java
Normal file
291
src/fr/devinsy/statoolinfos/metrics/etherpad/EtherpadLog.java
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
/*
|
||||
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.etherpad;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.metrics.TimeMark;
|
||||
import fr.devinsy.statoolinfos.metrics.TimeMarkUtils;
|
||||
import fr.devinsy.strings.StringList;
|
||||
|
||||
/**
|
||||
* The Class EtherpadLog.
|
||||
*/
|
||||
public class EtherpadLog
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(EtherpadLog.class);
|
||||
|
||||
private LocalDateTime time;
|
||||
private String level;
|
||||
private String type;
|
||||
private String event;
|
||||
private String padname;
|
||||
private String ip;
|
||||
private String author;
|
||||
|
||||
/**
|
||||
* Instantiates a new etherpad log.
|
||||
*/
|
||||
public EtherpadLog()
|
||||
{
|
||||
this.time = null;
|
||||
this.level = null;
|
||||
this.type = null;
|
||||
this.event = null;
|
||||
this.padname = null;
|
||||
this.ip = null;
|
||||
this.author = null;
|
||||
}
|
||||
|
||||
public String getAuthor()
|
||||
{
|
||||
return this.author;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the date.
|
||||
*
|
||||
* @return the date
|
||||
*/
|
||||
public String getDate()
|
||||
{
|
||||
String result;
|
||||
|
||||
result = this.time.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
public String getEvent()
|
||||
{
|
||||
return this.event;
|
||||
}
|
||||
|
||||
public String getIp()
|
||||
{
|
||||
return this.ip;
|
||||
}
|
||||
|
||||
public String getLevel()
|
||||
{
|
||||
return this.level;
|
||||
}
|
||||
|
||||
public String getPadname()
|
||||
{
|
||||
return this.padname;
|
||||
}
|
||||
|
||||
public LocalDateTime getTime()
|
||||
{
|
||||
return this.time;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the time marks.
|
||||
*
|
||||
* @return the time marks
|
||||
*/
|
||||
public StringList getTimeMarks()
|
||||
{
|
||||
StringList result;
|
||||
|
||||
result = new StringList();
|
||||
|
||||
String year = TimeMark.yearOf(this.time).toString();
|
||||
String yearMonth = TimeMark.yearMonthOf(this.time).toString();
|
||||
String yearWeek = TimeMark.yearWeekOf(this.time).toString();
|
||||
String date = TimeMark.dayOf(this.time).toString();
|
||||
|
||||
result.add(year);
|
||||
result.add(yearMonth);
|
||||
result.add(yearWeek);
|
||||
result.add(date);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
public String getType()
|
||||
{
|
||||
return this.type;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year.
|
||||
*
|
||||
* @return the year
|
||||
*/
|
||||
public String getYear()
|
||||
{
|
||||
String result;
|
||||
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearOf(this.time);
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year month.
|
||||
*
|
||||
* @return the year month
|
||||
*/
|
||||
public String getYearMonth()
|
||||
{
|
||||
String result;
|
||||
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearMonthOf(this.time);
|
||||
}
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the year week.
|
||||
*
|
||||
* @return the year week
|
||||
*/
|
||||
public String getYearWeek()
|
||||
{
|
||||
String result;
|
||||
if (this.time == null)
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = TimeMarkUtils.yearWeekOf(this.time);
|
||||
}
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if is ipv4.
|
||||
*
|
||||
* @return true, if is ipv4
|
||||
*/
|
||||
public boolean isIPv4()
|
||||
{
|
||||
boolean result;
|
||||
|
||||
if (this.ip == null)
|
||||
{
|
||||
result = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = this.ip.contains(".");
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if is ipv6.
|
||||
*
|
||||
* @return true, if is ipv6
|
||||
*/
|
||||
public boolean isIPv6()
|
||||
{
|
||||
boolean result;
|
||||
|
||||
if (this.ip == null)
|
||||
{
|
||||
result = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
result = this.ip.contains(":");
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
public void setAuthor(final String author)
|
||||
{
|
||||
this.author = author;
|
||||
}
|
||||
|
||||
public void setEvent(final String event)
|
||||
{
|
||||
this.event = event;
|
||||
}
|
||||
|
||||
public void setIp(final String ip)
|
||||
{
|
||||
this.ip = ip;
|
||||
}
|
||||
|
||||
public void setLevel(final String level)
|
||||
{
|
||||
this.level = level;
|
||||
}
|
||||
|
||||
public void setPadname(final String padname)
|
||||
{
|
||||
this.padname = padname;
|
||||
}
|
||||
|
||||
public void setTime(final LocalDateTime time)
|
||||
{
|
||||
this.time = time;
|
||||
}
|
||||
|
||||
public void setType(final String type)
|
||||
{
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see java.lang.Object#toString()
|
||||
*/
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
String result;
|
||||
|
||||
result = String.format("[time=%s, level=%s, type=%s, event=%s, %padname=%s, ip=%s, auhtor=%s]", this.time, this.level, this.type, this.event, this.padname, this.ip, this.author);
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,224 @@
|
|||
/*
|
||||
* Copyright (C) 2022 Christian Pierre MOMON <christian@momon.org>
|
||||
*
|
||||
* This file is part of StatoolInfos, simple service statistics tool.
|
||||
*
|
||||
* StatoolInfos is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* StatoolInfos is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package fr.devinsy.statoolinfos.metrics.etherpad;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import fr.devinsy.statoolinfos.metrics.PathCounters;
|
||||
import fr.devinsy.statoolinfos.metrics.UserCounters;
|
||||
import fr.devinsy.statoolinfos.util.Files;
|
||||
import fr.devinsy.statoolinfos.util.LineIterator;
|
||||
|
||||
/**
|
||||
* The Class EtherpadLogProber.
|
||||
*/
|
||||
public class EtherpadLogProber
|
||||
{
|
||||
private static Logger logger = LoggerFactory.getLogger(EtherpadLogProber.class);
|
||||
|
||||
// 2021-11-17 01:51:59 DELETE 7922dd508e0cf9b2
|
||||
public static final Pattern LOG_PATTERN = Pattern.compile(
|
||||
"^\\[(?<datetime>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(\\.\\d{3})?)]\\s\\[(?<level>[^]]+)]\\s+(?<type>\\S+) - \\[(?<event>[^]]+)]\\spad:(?<padname>\\S+)\\ssocket:\\S+\\sIP:(?<ip>\\S+) authorID:(?<author>\\S+)( username:.+)?$");
|
||||
/*
|
||||
[2022-03-28 02:19:35.842] [INFO] access - [ENTER] pad:test socket:Qr8BEX3mP6EmheyLAAAA IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
|
||||
[2022-03-28 18:50:10.963] [INFO] access - [CREATE] pad:test7 socket:WddndPw_4vt01-0YAAAC IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
|
||||
[2022-03-28 18:52:00.668] [INFO] access - [ENTER] pad:test7 socket:gA4Ba57988zrGJNmAAAD IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
|
||||
[2022-03-28 18:52:27.947] [INFO] access - [ENTER] pad:test7 socket:9ZrFheETmDUpUcS1AAAE IP:82.65.227.202 authorID:a.AMpqExl3JHbPL52c
|
||||
[2022-03-28 18:59:58.501] [INFO] access - [ENTER] pad:test socket:mCgPMxg2wb7upHP8AAAG IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
|
||||
[2022-03-28 19:01:24.884] [INFO] access - [LEAVE] pad:test socket:mCgPMxg2wb7upHP8AAAG IP:82.65.227.202 authorID:a.vZ7pRZcyfzUxO26i username:admin
|
||||
*/
|
||||
|
||||
public static final Pattern DELETE_EMPTY_LOG_PATTERN = Pattern.compile(
|
||||
"^\\[(?<datetime>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(\\.\\d{3})?)]\\s\\[(?<level>[^]]+)]\\s+(?<type>\\S+) - Deleting (?<padname>\\S+) when user leaved since empty$");
|
||||
/*
|
||||
[2022-03-29 00:43:57.085] [INFO] ep_delete_empty_pads - Deleting test8 when user leaved since empty
|
||||
*/
|
||||
|
||||
/**
|
||||
* Instantiates a new etherpad log prober.
|
||||
*/
|
||||
private EtherpadLogProber()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the log.
|
||||
*
|
||||
* @param line
|
||||
* the line
|
||||
* @return the http log
|
||||
*/
|
||||
public static EtherpadLog parseLog(final String line)
|
||||
{
|
||||
EtherpadLog result;
|
||||
|
||||
Matcher matcher = LOG_PATTERN.matcher(line);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = new EtherpadLog();
|
||||
result.setTime(LocalDateTime.parse(matcher.group("datetime"), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withLocale(Locale.ENGLISH)));
|
||||
result.setLevel(matcher.group("level"));
|
||||
result.setType(matcher.group("type"));
|
||||
result.setEvent(matcher.group("event"));
|
||||
result.setPadname(matcher.group("padname"));
|
||||
result.setIp(matcher.group("ip"));
|
||||
result.setAuthor(matcher.group("author"));
|
||||
}
|
||||
else
|
||||
{
|
||||
matcher = DELETE_EMPTY_LOG_PATTERN.matcher(line);
|
||||
if (matcher.matches())
|
||||
{
|
||||
result = new EtherpadLog();
|
||||
result.setTime(LocalDateTime.parse(matcher.group("datetime"), DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS").withLocale(Locale.ENGLISH)));
|
||||
result.setLevel(matcher.group("level"));
|
||||
result.setType(matcher.group("type"));
|
||||
result.setEvent("Deleting");
|
||||
result.setAuthor(matcher.group("padname"));
|
||||
}
|
||||
else
|
||||
{
|
||||
result = null;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe.
|
||||
*
|
||||
* @param file
|
||||
* the file
|
||||
* @return the path counters
|
||||
*/
|
||||
public static PathCounters probe(final Files files) throws IOException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
result = new PathCounters();
|
||||
|
||||
UserCounters users = new UserCounters();
|
||||
UserCounters ipv4Users = new UserCounters();
|
||||
UserCounters ipv6Users = new UserCounters();
|
||||
UserCounters actives = new UserCounters();
|
||||
|
||||
for (File file : files)
|
||||
{
|
||||
System.out.println("Probing file [" + file.getAbsolutePath() + "]");
|
||||
|
||||
//
|
||||
int lineCount = 0;
|
||||
LineIterator iterator = new LineIterator(file);
|
||||
while (iterator.hasNext())
|
||||
{
|
||||
lineCount += 1;
|
||||
String line = iterator.next();
|
||||
|
||||
// Remove ANSI Escape Codes.
|
||||
line = line.replaceAll("\u001B\\[[\\d;]*[^\\d;]", "");
|
||||
|
||||
try
|
||||
{
|
||||
EtherpadLog log = parseLog(line);
|
||||
// System.out.println("====> " + log);
|
||||
if (log == null)
|
||||
{
|
||||
if (line.contains("access -"))
|
||||
{
|
||||
logger.warn("LINE IS NOT MATCHING [{}]", line);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Timemarks.
|
||||
String year = log.getYear();
|
||||
String yearMonth = log.getYearMonth();
|
||||
String yearWeek = log.getYearWeek();
|
||||
String date = log.getDate();
|
||||
|
||||
if (StringUtils.equals(log.getType(), "access"))
|
||||
{
|
||||
// metrics.service.users
|
||||
// metrics.service.users.ipv4
|
||||
// metrics.service.users.ipv6
|
||||
users.put(log.getAuthor(), year, yearMonth, yearWeek, date);
|
||||
if (log.isIPv4())
|
||||
{
|
||||
ipv4Users.put(log.getAuthor(), year, yearMonth, yearWeek, date);
|
||||
}
|
||||
else
|
||||
{
|
||||
ipv6Users.put(log.getAuthor(), year, yearMonth, yearWeek, date);
|
||||
}
|
||||
|
||||
// metrics.etherpad.pads.actives
|
||||
actives.put(log.getPadname(), year, yearMonth, yearWeek, date);
|
||||
|
||||
// metrics.etherpad.pads.created
|
||||
// alias metrics.textprocessors.created
|
||||
if (StringUtils.equals(log.getEvent(), "CREATE"))
|
||||
{
|
||||
result.inc("metrics.etherpad.pads.created", year, yearMonth, yearWeek, date);
|
||||
result.inc("metrics.textprocessors.created", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
|
||||
if (StringUtils.equalsAny(log.getEvent(), "CREATE", "ENTER"))
|
||||
{
|
||||
result.inc("metrics.etherpad.pads.reads", year, yearMonth, yearWeek, date);
|
||||
result.inc("metrics.textprocessors.reads", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
}
|
||||
else if (StringUtils.equals(log.getType(), "ep_delete_empty_pads"))
|
||||
{
|
||||
result.inc("metrics.etherpad.pads.deleted", year, yearMonth, yearWeek, date);
|
||||
result.inc("metrics.textprocessors.deleted", year, yearMonth, yearWeek, date);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.warn("Error parsing line [{}][{}]", line, exception.getMessage());
|
||||
exception.printStackTrace();
|
||||
}
|
||||
}
|
||||
System.out.println("PrivatebinPatchLog line count=" + lineCount);
|
||||
}
|
||||
|
||||
//
|
||||
result.putAll(users.getCounters("metrics.service.users"));
|
||||
result.putAll(ipv4Users.getCounters("metrics.service.users.ipv4"));
|
||||
result.putAll(ipv6Users.getCounters("metrics.service.users.ipv6"));
|
||||
result.putAll(actives.getCounters("metrics.etherpad.pads.actives"));
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -61,14 +61,21 @@ public class EtherpadProber
|
|||
* @throws StatoolInfosException
|
||||
* the statool infos exception
|
||||
*/
|
||||
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException
|
||||
public static PathCounters probe(final Files httpLogs, final String httpLogRegex, final Files logs, final DatabaseConfig databaseConfig) throws IOException, StatoolInfosException
|
||||
{
|
||||
PathCounters result;
|
||||
|
||||
// metrics.service.users
|
||||
// metrics.service.users.ipv4
|
||||
// metrics.service.users.ipv6
|
||||
if (logs.isEmpty())
|
||||
{
|
||||
result = EtherpadHttpLogAnalyzer.probe(httpLogs, httpLogRegex);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = EtherpadLogProber.probe(logs);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
|
|
@ -144,10 +151,6 @@ public class EtherpadProber
|
|||
logger.error("ERROR with database.", exception);
|
||||
}
|
||||
|
||||
/*
|
||||
* select substring(key, strpos(key, ':')+1, strpos(substring(key, strpos(key, ':')+1), ':')) from store where key like 'pad:%:revs:%';
|
||||
*/
|
||||
|
||||
//
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue