Step in probing dev.

This commit is contained in:
Christian P. MOMON 2020-10-15 01:47:48 +02:00
parent a889e629ff
commit 628865f790
11 changed files with 852 additions and 5 deletions

View file

@ -101,9 +101,10 @@ public final class StatoolInfosCLI
message.appendln("Usage:");
message.appendln(" statoolinfos [ -h | -help | --help ]");
message.appendln(" statoolinfos [ -v | -version | --version ]");
message.appendln(" statoolinfos clear [ directory | file ]");
message.appendln(" statoolinfos build [ directory | file ]");
message.appendln(" statoolinfos crawl [ directory | file ]");
message.appendln(" statoolinfos clear [ directory | file ]");
message.appendln(" statoolinfos probe [ directory | file ]");
message.appendln(" statoolinfos build [ directory | file ]");
message.appendln(" statoolinfos crawl [ directory | file ]");
message.appendln(" statoolinfos htmlize [ directory | file ]");
logger.info(message.toString());
@ -241,6 +242,21 @@ public final class StatoolInfosCLI
}
}
}
else if (isMatching(args, "probe", "\\s*.+\\s*"))
{
Files inputs = convertPath(StringUtils.trim(args[1]));
for (File input : inputs)
{
try
{
StatoolInfos.probe(input);
}
catch (Exception exception)
{
logger.error("Error with [{}]: {}", input.getAbsoluteFile(), exception.getMessage());
}
}
}
else if (isMatching(args, "build", "\\s*.+\\s*"))
{
Files inputs = convertPath(StringUtils.trim(args[1]));

View file

@ -37,6 +37,7 @@ public class Metric
DAYS
}
private String path;
private String name;
private String description;
private String startYear;
@ -48,8 +49,9 @@ public class Metric
/**
* Instantiates a new metric.
*/
public Metric(final String name, final String description, final String startYear)
public Metric(final String path, final String name, final String description, final String startYear)
{
this.path = path;
this.name = name;
this.description = description;
this.startYear = startYear;

View file

@ -233,7 +233,7 @@ public class Service extends PathPropertyList
}
else
{
result = new Metric(metricName, metricDescription, years.get(0));
result = new Metric(path, metricName, metricDescription, years.get(0));
for (String year : years)
{
@ -360,6 +360,21 @@ public class Service extends PathPropertyList
return result;
}
/**
* Gets the software technical name.
*
* @return the software technical name
*/
public String getSoftwareTechnicalName()
{
String result;
result = StatoolInfosUtils.toTechnicalName(getSoftwareName());
//
return result;
}
/**
* Gets the software version.
*

View file

@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.build.Builder;
import fr.devinsy.statoolinfos.crawl.Crawler;
import fr.devinsy.statoolinfos.htmlize.Htmlizer;
import fr.devinsy.statoolinfos.htmlize.probes.Prober;
/**
* The Class StatoolInfos.
@ -99,4 +100,19 @@ public class StatoolInfos
{
Htmlizer.htmlize(configurationFile);
}
/**
* Stat.
*
* @param configurationFile
* the configuration file
* @throws StatoolInfosException
* the statool infos exception
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void probe(final File configurationFile) throws StatoolInfosException, IOException
{
Prober.probe(configurationFile);
}
}

View file

@ -0,0 +1,238 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.htmlize.probes;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
/**
* The Class HttpAccessLogProber.
*/
public class HttpAccessLogProber
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogProber.class);
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
/**
* Instantiates a new http access log prober.
*/
private HttpAccessLogProber()
{
}
/**
* Stat.
*
* @param configuration
* the configuration
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static void probe(final File source, final File target) throws IOException, StatoolInfosException
{
PathCounters counters;
counters = new PathCounters();
for (File file : source.getParentFile().listFiles())
{
if (file.getName().startsWith(source.getName()))
{
probe(counters, source);
}
}
}
/**
* Probe.
*
* @param source
* the source
* @param counters
* the counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static void probe(final PathCounters counters, final File source) throws IOException, StatoolInfosException
{
BufferedReader in = null;
try
{
in = new BufferedReader(new InputStreamReader(new FileInputStream(source), DEFAULT_CHARSET_NAME));
boolean ended = false;
while (!ended)
{
String line = in.readLine();
if (line == null)
{
ended = true;
}
else if (StringUtils.isNotEmpty(line))
{
probe(counters, line);
}
}
}
finally
{
IOUtils.closeQuietly(in);
}
}
/**
* Probe.
*
* @param counter
* the counter
* @param line
* the line
*/
public static void probe(final PathCounters counters, final String line)
{
//
LocalDateTime date;
date = LocalDateTime.now();
String year = date.format(DateTimeFormatter.ofPattern("yyyy", Locale.FRANCE));
String month = date.format(DateTimeFormatter.ofPattern("yyyy-MM", Locale.FRANCE));
String week = date.format(DateTimeFormatter.ofPattern("yyyyWW", Locale.FRANCE));
String day = date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
// General HTTP access logs.
// metrics.http.hits
counters.inc("metrics.http.hits", year);
counters.inc("metrics.http.hits", month);
counters.inc("metrics.http.hits", week);
counters.inc("metrics.http.hits", day);
// metrics.http.hits.ipv4
counters.inc("metrics.http.hits.ipv4", year);
counters.inc("metrics.http.hits.ipv4", month);
counters.inc("metrics.http.hits.ipv4", week);
counters.inc("metrics.http.hits.ipv4", day);
// metrics.http.hits.ipv6
counters.inc("metrics.http.hits.ipv6", year);
counters.inc("metrics.http.hits.ipv6", month);
counters.inc("metrics.http.hits.ipv6", week);
counters.inc("metrics.http.hits.ipv6", day);
// metrics.http.pages
counters.inc("metrics.http.pages", year);
counters.inc("metrics.http.pages", month);
counters.inc("metrics.http.pages", week);
counters.inc("metrics.http.pages", day);
// metrics.http.pages.ipv4
counters.inc("metrics.http.pages.ipv4", year);
counters.inc("metrics.http.pages.ipv4", month);
counters.inc("metrics.http.pages.ipv4", week);
counters.inc("metrics.http.pages.ipv4", day);
// metrics.http.pages.ipv6
counters.inc("metrics.http.pages.ipv6", year);
counters.inc("metrics.http.pages.ipv6", month);
counters.inc("metrics.http.pages.ipv6", week);
counters.inc("metrics.http.pages.ipv6", day);
// metrics.http.files
counters.inc("metrics.http.file", year);
counters.inc("metrics.http.file", month);
counters.inc("metrics.http.file", week);
counters.inc("metrics.http.file", day);
// metrics.http.files.ipv4
counters.inc("metrics.http.file.ipv4", year);
counters.inc("metrics.http.file.ipv4", month);
counters.inc("metrics.http.file.ipv4", week);
counters.inc("metrics.http.file.ipv4", day);
// metrics.http.files.ipv6
counters.inc("metrics.http.file.ipv6", year);
counters.inc("metrics.http.file.ipv6", month);
counters.inc("metrics.http.file.ipv6", week);
counters.inc("metrics.http.file.ipv6", day);
// metrics.http.bytes
counters.inc("metrics.http.bytes", year);
counters.inc("metrics.http.bytes", month);
counters.inc("metrics.http.bytes", week);
counters.inc("metrics.http.bytes", day);
// metrics.http.bytes.ipv4
counters.inc("metrics.http.bytes.ipv4", year);
counters.inc("metrics.http.bytes.ipv4", month);
counters.inc("metrics.http.bytes.ipv4", week);
counters.inc("metrics.http.bytes.ipv4", day);
// metrics.http.bytes.ipv6
counters.inc("metrics.http.bytes.ipv6", year);
counters.inc("metrics.http.bytes.ipv6", month);
counters.inc("metrics.http.bytes.ipv6", week);
counters.inc("metrics.http.bytes.ipv6", day);
// metrics.http.status.1xx
// metrics.http.status.2xx
// metrics.http.status.3xx
// metrics.http.status.4xx
// metrics.http.status.5xx
// metrics.http.ips
// metrics.http.ips.ipv4
// metrics.http.ips.ipv6
// General HTTP error logs.
// metrics.http.errors
// metrics.http.errors.php
// ==
// metrics.http.hits.bots
// metrics.http.hits.monitoring
// metrics.http.hits.
// metrics.http.monitoring
// metrics.http.visitors
// metrics.http.users
}
}

View file

@ -0,0 +1,63 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.htmlize.probes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class HttpAccesLogStat.
*/
public class HttpAccessLogStat
{
private static Logger logger = LoggerFactory.getLogger(HttpAccessLogStat.class);
private int hitCount;
private int hitIpv4Count;
private int hitIpv6Count;
private int pageCount;
private int pageIpv4Count;
private int pageIpv6Count;
private int byteCount;
private int byteIpv4Count;
private int byteIpv6Count;
private int fileCount;
private int fileIpv4Count;
private int fileIpv6Count;
private int status1xxCount;
private int status2xxCount;
private int status3xxCount;
private int status4xxCount;
private int status5xxCount;
private int ipCount;
private int ipIpv4Count;
private int ipIpv6Count;
/**
* Instantiates a new http acces log stat.
*/
public HttpAccessLogStat()
{
}
}

View file

@ -0,0 +1,152 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.htmlize.probes;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
/**
* The Class HttpErrorLogProber.
*/
public class HttpErrorLogProber
{
private static Logger logger = LoggerFactory.getLogger(HttpErrorLogProber.class);
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
/**
* Instantiates a new http error log prober.
*/
private HttpErrorLogProber()
{
}
/**
* Stat.
*
* @param configuration
* the configuration
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static void probe(final File source, final File target) throws IOException, StatoolInfosException
{
PathCounters counters;
counters = new PathCounters();
for (File file : source.getParentFile().listFiles())
{
if (file.getName().startsWith(source.getName()))
{
probe(counters, source);
}
}
}
/**
* Probe.
*
* @param source
* the source
* @param counters
* the counters
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static void probe(final PathCounters counters, final File source) throws IOException, StatoolInfosException
{
BufferedReader in = null;
try
{
in = new BufferedReader(new InputStreamReader(new FileInputStream(source), DEFAULT_CHARSET_NAME));
boolean ended = false;
while (!ended)
{
String line = in.readLine();
if (line == null)
{
ended = true;
}
else if (StringUtils.isNotEmpty(line))
{
probe(counters, line);
}
}
}
finally
{
IOUtils.closeQuietly(in);
}
}
/**
* Probe.
*
* @param counter
* the counter
* @param line
* the line
*/
public static void probe(final PathCounters counters, final String line)
{
//
LocalDateTime date;
date = LocalDateTime.now();
String year = date.format(DateTimeFormatter.ofPattern("yyyy", Locale.FRANCE));
String month = date.format(DateTimeFormatter.ofPattern("yyyy-MM", Locale.FRANCE));
String week = date.format(DateTimeFormatter.ofPattern("yyyyWW", Locale.FRANCE));
String day = date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd", Locale.FRANCE));
// General HTTP access logs.
// metrics.http.errors
counters.inc("metrics.http.errors", year);
counters.inc("metrics.http.errors", month);
counters.inc("metrics.http.errors", week);
counters.inc("metrics.http.errors", day);
// metrics.http.hits.ipv4
counters.inc("metrics.http.errors.php", year);
counters.inc("metrics.http.errors.php", month);
counters.inc("metrics.http.errors.php", week);
counters.inc("metrics.http.errors.php", day);
}
}

View file

@ -0,0 +1,87 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.htmlize.probes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class PathCounter.
*/
public class PathCounter
{
private static Logger logger = LoggerFactory.getLogger(PathCounter.class);
private String path;
private String timeMark;
private long counter;
/**
* Instantiates a new path counter.
*
* @param path
* the path
* @param timeMark
* the time mark
*/
public PathCounter(final String path, final String timeMark)
{
this.path = path;
this.timeMark = timeMark;
this.counter = 0;
}
public long getCounter()
{
return this.counter;
}
public String getPath()
{
return this.path;
}
public String getTimeMark()
{
return this.timeMark;
}
/**
* Inc.
*/
public void inc()
{
this.counter += 1;
}
public void setCounter(final long counter)
{
this.counter = counter;
}
public void setPath(final String path)
{
this.path = path;
}
public void setTimeMark(final String timeMark)
{
this.timeMark = timeMark;
}
}

View file

@ -0,0 +1,97 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.htmlize.probes;
import java.util.HashMap;
/**
* The Class PathCounters.
*/
public class PathCounters extends HashMap<String, PathCounter>
{
private static final long serialVersionUID = -7731827840828688703L;
/**
* Instantiates a new path counters.
*/
public PathCounters()
{
super();
}
/**
* Compute key.
*
* @param path
* the path
* @param timeMark
* the time mark
* @return the string
*/
public String computeKey(final String path, final String timeMark)
{
String result;
result = path + "." + timeMark;
//
return result;
}
/**
* Gets the.
*
* @param path
* the path
* @param timeMark
* the time mark
* @return the path counter
*/
public PathCounter get(final String path, final String timeMark)
{
PathCounter result;
String key = computeKey(path, timeMark);
result = get(key);
//
return result;
}
/**
* Put.
*
* @param path
* the path
* @param timeMark
* the time mark
*/
public void inc(final String path, final String timeMark)
{
PathCounter counter = get(path, timeMark);
if (counter == null)
{
counter = new PathCounter(path, timeMark);
put(computeKey(path, timeMark), counter);
}
counter.inc();
}
}

View file

@ -0,0 +1,110 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.htmlize.probes;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.Configuration;
import fr.devinsy.statoolinfos.core.Factory;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
/**
* The Class Prober.
*/
public class Prober
{
private static Logger logger = LoggerFactory.getLogger(Prober.class);
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
/**
* Instantiates a new prober.
*/
private Prober()
{
}
/**
* Probe.
*
* @param configuration
* the configuration
* @throws IOException
* Signals that an I/O exception has occurred.
* @throws StatoolInfosException
* the statool infos exception
*/
public static void probe(final Configuration configuration) throws IOException, StatoolInfosException
{
BufferedReader in = null;
try
{
//
{
String sourceFileName = configuration.get("conf.probe.http.accesslogs.source");
String targetFileName = configuration.get("conf.probe.http.accesslogs.target");
if (!StringUtils.isAllBlank(sourceFileName, targetFileName))
{
HttpAccessLogProber.probe(new File(sourceFileName), new File(targetFileName));
}
}
//
// {
// String sourceFileName =
// configuration.get("conf.probe.httperrorlogs.source");
// String targetFileName =
// configuration.get("conf.probe.httperrorlogs.target");
// if (!StringUtils.isAllBlank(sourceFileName, targetFileName))
// {
// HttpErrorLogProber.probe(sourceFileName, targetFileName);
// }
// }
}
finally
{
IOUtils.closeQuietly(in);
}
}
/**
* Stat.
*
* @param configurationFile
* the configuration file
* @throws IOException
* @throws StatoolInfosException
*/
public static void probe(final File configurationFile) throws StatoolInfosException, IOException
{
logger.info("Htmlize {}", configurationFile.getAbsolutePath());
Configuration configuration = Factory.loadConfiguration(configurationFile);
probe(configuration);
}
}

View file

@ -19,6 +19,8 @@
package fr.devinsy.statoolinfos.stats.properties;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.properties.PathProperty;
import fr.devinsy.statoolinfos.properties.PathPropertyList;
@ -28,6 +30,8 @@ import fr.devinsy.statoolinfos.properties.PathPropertyList;
*/
public class PropertyStats
{
private static Logger logger = LoggerFactory.getLogger(PropertyStats.class);
private int fileCount;
private PropertyStatSet stats;
@ -67,6 +71,29 @@ public class PropertyStats
return this.fileCount;
}
/**
* Gets the not metric property stats.
*
* @return the not metric property stats
*/
public PropertyStats getGeneralPropertyStats()
{
PropertyStats result;
result = new PropertyStats();
for (PropertyStat stat : this.stats.values())
{
if (!StringUtils.startsWithAny(stat.getPath(), "metrics.", "subs."))
{
result.stats.put(stat);
}
}
//
return result;
}
/**
* @return
*/
@ -85,6 +112,29 @@ public class PropertyStats
return result;
}
/**
* Gets the metric property stats.
*
* @return the metric property stats
*/
public PropertyStats getMetricPropertyStats()
{
PropertyStats result;
result = new PropertyStats();
for (PropertyStat stat : this.stats.values())
{
if (StringUtils.startsWithAny(stat.getPath(), "metrics.", "subs."))
{
result.stats.put(stat);
}
}
//
return result;
}
public int getPropertyCount()
{
return this.stats.size();
@ -131,4 +181,5 @@ public class PropertyStats
this.stats.put(stat);
}
}
}