statool/statoolinfos
statool/statoolinfos
1
0
Fork 0
Code Issues Pull requests Releases 8 Activity

Added metrics property file management.

Browse source
This commit is contained in:
Christian P. MOMON 2021-05-20 16:03:29 +02:00
parent d8e2aa2cec
commit 0793a616f6
12 changed files with 493 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
src/fr/devinsy/statoolinfos/checker/PropertyChecker.java
View file

@ -57,9 +57,10 @@ public class PropertyChecker
public static final String METRICS_DAYS = "^metrics\\.\\S+\\.\\d{4}\\.days$";
public static final String CRAWL = "^crawl\\.\\S+$";
private PropertyRules serviceRules;
private PropertyRules federationRules;
private PropertyRules organizationRules;
private PropertyRules serviceRules;
private PropertyRules metricsRules;
/**
* Instantiates a new property checker.
@ -190,6 +191,23 @@ public class PropertyChecker
this.serviceRules.add(METRICS_DAYS, DAYS, PropertyMode.OPTIONAL);
this.serviceRules.add(CRAWL, ALL, PropertyMode.MANDATORY);
//
this.metricsRules = new PropertyRules();
this.metricsRules.add("file.class", "^[Mm]etrics$", PropertyMode.MANDATORY);
this.metricsRules.add("file.generator", STRING, PropertyMode.WISHED);
this.metricsRules.add("file.datetime", DATETIME, PropertyMode.WISHED);
this.metricsRules.add("file.protocol", STRING, PropertyMode.WISHED);
this.metricsRules.add(METRICS_NAME, STRING, PropertyMode.OPTIONAL);
this.metricsRules.add(METRICS_DESCRIPTION, STRING, PropertyMode.OPTIONAL);
this.metricsRules.add(METRICS_YEAR, NUMERIC, PropertyMode.OPTIONAL);
this.metricsRules.add(METRICS_MONTHS, MONTHS, PropertyMode.OPTIONAL);
this.metricsRules.add(METRICS_WEEKS, WEEKS, PropertyMode.OPTIONAL);
this.metricsRules.add(METRICS_DAYS, DAYS, PropertyMode.OPTIONAL);
this.metricsRules.add(CRAWL, ALL, PropertyMode.MANDATORY);
}
/**
@ -365,6 +383,26 @@ public class PropertyChecker
return result;
}
/**
* Check metrics.
*
* @param file
* the file
* @return the property checks
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public PropertyChecks checkMetrics(final File file) throws IOException
{
PropertyChecks result;
StringList lines = StringsUtils.load(file);
result = check(lines, this.metricsRules);
//
return result;
}
/**
* Check organization.
*

81
src/fr/devinsy/statoolinfos/core/Factory.java
View file

@ -226,6 +226,49 @@ public class Factory
return result;
}
/**
* Load metrics.
*
* @param inputURL
* the input URL
* @param cache
* the cache
* @param prefixPath
* the prefix path
* @return the metrics
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Metrics loadMetrics(final URL inputURL, final CrawlCache cache, final String prefixPath) throws IOException
{
Metrics result;
File inputFile = cache.restoreFile(inputURL);
if (inputFile == null)
{
logger.warn("WARNING: metrics not found in cache [{}]", inputURL);
result = null;
}
else
{
PathProperties properties = PathPropertyUtils.load(inputFile);
result = new Metrics(properties);
result.setLocalFileNamePrefix(prefixPath);
result.setInputFile(inputFile);
result.setInputURL(inputURL);
//
PropertyChecker checker = new PropertyChecker();
PropertyChecks checks = checker.checkMetrics(result.getInputFile());
result.getInputChecks().addAll(checks);
result.getInputChecks().setFileName(result.getLocalFileName());
}
//
return result;
}
/**
* Load organization.
*
@ -261,25 +304,19 @@ public class Factory
//
PropertyChecker checker = new PropertyChecker();
PropertyChecks checks = checker.checkOrganization(result.getInputFile());
checks.setFileName(result.getLocalFileName());
result.getInputChecks().addAll(checks);
result.getInputChecks().setFileName(result.getLocalFileName());
//
PathProperties subs = result.getByPrefix("subs");
for (PathProperty property : subs)
{
if (StringUtils.startsWith(property.getValue(), "http"))
{
URL serviceInputURL = new URL(property.getValue());
Service service = loadService(serviceInputURL, cache);
Service service = loadService(serviceInputURL, cache, result);
if (service != null)
{
service.setOrganization(result);
service.setLogoFileName(result.getTechnicalName() + "-" + service.getLogoFileName());
PropertyChecks subChecks = checker.checkService(service.getInputFile());
service.getInputChecks().addAll(subChecks);
service.getInputChecks().setFileName(service.getLocalFileName());
result.getServices().add(service);
}
}
@ -311,7 +348,7 @@ public class Factory
* @return the service
* @throws IOException
*/
public static Service loadService(final URL inputURL, final CrawlCache cache) throws IOException
public static Service loadService(final URL inputURL, final CrawlCache cache, final Organization organization) throws IOException
{
Service result;
@ -326,9 +363,31 @@ public class Factory
{
PathProperties properties = PathPropertyUtils.load(inputFile);
result = new Service(properties);
result.setOrganization(organization);
result.setInputFile(inputFile);
result.setInputURL(inputURL);
result.setLogoFileName(result.getTechnicalName() + "-logo" + StringUtils.defaultIfBlank(cache.getExtension(result.getLogoURL()), ".png"));
result.setLogoFileName(organization.getTechnicalName() + "-" + result.getTechnicalName() + "-logo" + StringUtils.defaultIfBlank(cache.getExtension(result.getLogoURL()), ".png"));
//
PathProperties subs = result.getByPrefix("subs");
for (PathProperty property : subs)
{
if (StringUtils.startsWith(property.getValue(), "http"))
{
URL metricsInputURL = new URL(property.getValue());
Metrics metrics = loadMetrics(metricsInputURL, cache, organization.getTechnicalName() + "-" + result.getTechnicalName());
if (metrics != null)
{
result.getMetrics().add(metrics);
}
}
}
//
PropertyChecker checker = new PropertyChecker();
PropertyChecks checks = checker.checkService(result.getInputFile());
checks.setFileName(result.getLocalFileName());
result.getInputChecks().addAll(checks);
//
CrawlJournal journal = cache.restoreJournal();

182
src/fr/devinsy/statoolinfos/core/Metrics.java Normal file
View file

@ -0,0 +1,182 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.core;
import java.io.File;
import java.net.URL;
import java.time.LocalDateTime;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.checker.PropertyChecks;
import fr.devinsy.statoolinfos.properties.PathProperties;
import fr.devinsy.statoolinfos.properties.PathPropertyList;
/**
* The Class Metrics.
*/
public class Metrics extends PathPropertyList
{
private static final long serialVersionUID = -1608084706095266037L;
private static Logger logger = LoggerFactory.getLogger(Metrics.class);
private String localFileNamePrefix;
private File inputFile;
private URL inputURL;
private PropertyChecks inputChecks;
/**
* Instantiates a new service.
*/
public Metrics()
{
this(null);
}
/**
* Instantiates a new service.
*
* @param properties
* the properties
*/
public Metrics(final PathProperties properties)
{
super(properties);
this.inputChecks = new PropertyChecks();
}
/**
* Gets the crawl date.
*
* @return the crawl date
*/
public LocalDateTime getCrawlDate()
{
LocalDateTime result;
result = LocalDateTime.parse(get("crawl.datetime"));
//
return result;
}
/**
* Gets the crawled date.
*
* @return the crawled date
*/
public LocalDateTime getCrawledDate()
{
LocalDateTime result;
result = LocalDateTime.parse(get("crawl.file.datetime"));
if (result.getYear() == 1970)
{
result = getCrawlDate();
}
//
return result;
}
public PropertyChecks getInputChecks()
{
return this.inputChecks;
}
public File getInputFile()
{
return this.inputFile;
}
public URL getInputURL()
{
return this.inputURL;
}
/**
* Gets the local file base name.
*
* @return the local file base name
*/
public String getLocalFileBaseName()
{
String result;
result = this.localFileNamePrefix + "-" + getTechnicalName();
//
return result;
}
/**
* Gets the local name.
*
* @return the local name
*/
public String getLocalFileName()
{
String result;
result = getLocalFileBaseName() + ".properties";
//
return result;
}
public String getLocalFileNamePrefix()
{
return this.localFileNamePrefix;
}
/**
* Gets the technical name.
*
* @return the technical name
*/
public String getTechnicalName()
{
String result;
String name = FilenameUtils.getBaseName(this.inputURL.getPath());
result = StatoolInfosUtils.toTechnicalName(name);
//
return result;
}
public void setInputFile(final File inputFile)
{
this.inputFile = inputFile;
}
public void setInputURL(final URL inputURL)
{
this.inputURL = inputURL;
}
public void setLocalFileNamePrefix(final String localFileNamePrefix)
{
this.localFileNamePrefix = localFileNamePrefix;
}
}

55
src/fr/devinsy/statoolinfos/core/MetricsList.java Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.core;
import java.util.ArrayList;
import java.util.Collections;
/**
* The Class MetricsList.
*/
public class MetricsList extends ArrayList<Metrics>
{
private static final long serialVersionUID = 6241734209264875306L;
/**
* Instantiates a new metrics list.
*/
public MetricsList()
{
super();
}
/**
* Reverse.
*
* @return the services
*/
public MetricsList reverse()
{
MetricsList result;
Collections.reverse(this);
result = this;
//
return result;
}
}

19
src/fr/devinsy/statoolinfos/core/Organization.java
View file

@ -253,7 +253,7 @@ public class Organization extends PathPropertyList
result.addAll(this.inputChecks);
for (Service service : this.services)
{
result.addAll(service.getInputChecks());
result.addAll(service.getInputChecksAll());
}
//
@ -295,6 +295,21 @@ public class Organization extends PathPropertyList
return result;
}
/**
* Gets the local file base name.
*
* @return the local file base name
*/
public String getLocalFileBaseName()
{
String result;
result = getTechnicalName();
//
return result;
}
/**
* Gets the local file name.
*
@ -304,7 +319,7 @@ public class Organization extends PathPropertyList
{
String result;
result = getTechnicalName() + ".properties";
result = getLocalFileBaseName() + ".properties";
//
return result;

9
src/fr/devinsy/statoolinfos/core/PropertyClassType.java
View file

@ -71,7 +71,14 @@ public enum PropertyClassType
}
break;
case METRICS:
result = true;
if (parent == METRICS)
{
result = false;
}
else
{
result = true;
}
break;
default:
result = false;

61
src/fr/devinsy/statoolinfos/core/Service.java
View file

@ -104,6 +104,7 @@ public class Service extends PathPropertyList
private File inputFile;
private URL inputURL;
private String logoFileName;
private MetricsList metricsList;
private PropertyChecks inputChecks;
private CrawlJournal crawlJournal;
@ -126,6 +127,7 @@ public class Service extends PathPropertyList
super(properties);
this.inputChecks = new PropertyChecks();
this.crawlJournal = new CrawlJournal();
this.metricsList = new MetricsList();
}
/**
@ -329,6 +331,28 @@ public class Service extends PathPropertyList
return this.inputChecks;
}
/**
* Gets the input checks all.
*
* @return the input checks all
*/
public PropertyChecks getInputChecksAll()
{
PropertyChecks result;
result = new PropertyChecks();
result.addAll(this.inputChecks);
for (Metrics metrics : getMetrics())
{
result.addAll(metrics.getInputChecks());
}
//
return result;
}
public File getInputFile()
{
return this.inputFile;
@ -369,6 +393,21 @@ public class Service extends PathPropertyList
return result;
}
/**
* Gets the local name.
*
* @return the local name
*/
public String getLocalFileBaseName()
{
String result;
result = this.organization.getTechnicalName() + "-" + getTechnicalName();
//
return result;
}
/**
* Gets the local name.
*
@ -378,7 +417,7 @@ public class Service extends PathPropertyList
{
String result;
result = this.organization.getTechnicalName() + "-" + getTechnicalName() + ".properties";
result = getLocalFileBaseName() + ".properties";
//
return result;
@ -451,6 +490,11 @@ public class Service extends PathPropertyList
return result;
}
public MetricsList getMetrics()
{
return this.metricsList;
}
/**
* Gets the metric years.
*
@ -509,21 +553,6 @@ public class Service extends PathPropertyList
return this.organization;
}
/**
* Gets the property file name.
*
* @return the property file name
*/
public String getPropertyFileName()
{
String result;
result = getOrganization().getTechnicalName() + "-" + getTechnicalName() + ".properties";
//
return result;
}
/**
* Gets the registration load.
*

28
src/fr/devinsy/statoolinfos/htmlize/PropertyFileCheckPage.java
View file

@ -30,6 +30,7 @@ import fr.devinsy.statoolinfos.HtmlizerContext;
import fr.devinsy.statoolinfos.checker.PropertyCheck;
import fr.devinsy.statoolinfos.checker.PropertyChecks;
import fr.devinsy.statoolinfos.core.Federation;
import fr.devinsy.statoolinfos.core.Metrics;
import fr.devinsy.statoolinfos.core.Organization;
import fr.devinsy.statoolinfos.core.Service;
import fr.devinsy.statoolinfos.core.Service.Status;
@ -75,26 +76,35 @@ public class PropertyFileCheckPage
{
checks = organization.getInputChecks();
page = htmlize("Organisation", checks);
FileUtils.write(new File(htmlizeDirectory, organization.getTechnicalName() + "-check.xhtml"), page, StandardCharsets.UTF_8);
FileUtils.write(new File(htmlizeDirectory, organization.getLocalFileBaseName() + "-check.xhtml"), page, StandardCharsets.UTF_8);
// Ignore ghost organizations.
if (organization.getServiceCount() > 0)
{
alerts = checks.getAlertLines().setFileName(organization.getLocalFileName());
allAlertChecks.addAll(alerts);
organizationAlertChecks.addAll(alerts);
}
for (Service service : organization.getServices())
{
checks = service.getInputChecks();
page = htmlize("Service", checks);
FileUtils.write(new File(htmlizeDirectory, organization.getTechnicalName() + "-" + service.getTechnicalName() + "-check.xhtml"), page, StandardCharsets.UTF_8);
if (organization.getServiceCount() > 0)
for (Service service : organization.getServices())
{
checks = service.getInputChecksAll();
page = htmlize("Service", checks);
FileUtils.write(new File(htmlizeDirectory, service.getLocalFileBaseName() + "-check.xhtml"), page, StandardCharsets.UTF_8);
alerts = checks.getAlertLines().setFileName(service.getLocalFileName());
allAlertChecks.addAll(alerts);
serviceAlertChecks.addAll(alerts);
for (Metrics metrics : service.getMetrics())
{
checks = metrics.getInputChecks();
page = htmlize("Metrics", checks);
FileUtils.write(new File(htmlizeDirectory, metrics.getLocalFileBaseName() + "-check.xhtml"), page, StandardCharsets.UTF_8);
alerts = checks.getAlertLines().setFileName(metrics.getLocalFileName());
allAlertChecks.addAll(alerts);
serviceAlertChecks.addAll(alerts);
}
}
}
}

14
src/fr/devinsy/statoolinfos/htmlize/ServicePage.java
View file

@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.catgenerator.core.BirdGenerator;
import fr.devinsy.statoolinfos.HtmlizerContext;
import fr.devinsy.statoolinfos.checker.PropertyChecks;
import fr.devinsy.statoolinfos.core.Metrics;
import fr.devinsy.statoolinfos.core.Organization;
import fr.devinsy.statoolinfos.core.Service;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
@ -67,12 +68,19 @@ public class ServicePage
//
logger.info("Htmlize service properties file: {}.", service.getName());
FileUtils.copyFile(service.getInputFile(),
new File(htmlizeDirectory, service.getOrganization().getTechnicalName() + "-" + service.getTechnicalName() + ".properties"));
new File(htmlizeDirectory, service.getLocalFileBaseName() + ".properties"));
//
for (Metrics metrics : service.getMetrics())
{
FileUtils.copyFile(metrics.getInputFile(),
new File(htmlizeDirectory, metrics.getLocalFileBaseName() + ".properties"));
}
//
logger.info("Htmlize service page: {}.", service.getName());
String page = ServicePage.htmlize(service.getOrganization(), service);
FileUtils.write(new File(htmlizeDirectory, service.getOrganization().getTechnicalName() + "-" + service.getTechnicalName() + ".xhtml"), page, StandardCharsets.UTF_8);
FileUtils.write(new File(htmlizeDirectory, service.getLocalFileBaseName() + ".xhtml"), page, StandardCharsets.UTF_8);
}
/**
@ -232,7 +240,7 @@ public class ServicePage
}
{
PropertyChecks checks = service.getInputChecks();
PropertyChecks checks = service.getInputChecksAll();
data.setContent("errorCount", checks.getErrorCount());
data.setContent("warningCount", checks.getWarningCount());
data.setContent("voidCount", checks.getVoidCount());

6
src/fr/devinsy/statoolinfos/htmlize/propertyFileCheck.xhtml
View file

@ -31,7 +31,7 @@
<tbody>
<tr id="line">
<td id="lineIndex" style="width: 30px; text-align: right; padding-right: 5px;">000</td>
<td id="lineContent" style="padding-left: 10px;">n/a</td>
<td id="lineContent" style="padding-left: 10px; max-width: 800px; word-wrap: anywhere;">n/a</td>
<td id="lineComment" style="width: 200px; padding-left: 5px;">n/a</td>
</tr>
</tbody>
@ -42,7 +42,7 @@
<tbody>
<tr id="shrunkLine">
<td id="shrunkLineIndex" style="width: 30px; text-align: right; right; padding-right: 5px;">000</td>
<td id="shrunkLineContent" style="padding-left: 10px;">n/a</td>
<td id="shrunkLineContent" style="padding-left: 10px; max-width: 800px; word-wrap: anywhere;">n/a</td>
<td id="shrunkLineComment" style="width: 200px; padding-left: 5px;">n/a</td>
</tr>
</tbody>
@ -53,7 +53,7 @@
<tbody>
<tr id="alertLine">
<td id="alertLineIndex" style="width: 30px; text-align: right; right; padding-right: 5px;">000</td>
<td id="alertLineContent" style="padding-left: 10px;">n/a</td>
<td id="alertLineContent" style="padding-left: 10px; max-width: 800px; word-wrap: anywhere;">n/a</td>
<td id="alertLineComment" style="width: 200px; padding-left: 5px;">n/a</td>
</tr>
</tbody>

16
src/fr/devinsy/statoolinfos/stats/StatAgent.java
View file

@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.Categories;
import fr.devinsy.statoolinfos.core.Category;
import fr.devinsy.statoolinfos.core.Federation;
import fr.devinsy.statoolinfos.core.Metrics;
import fr.devinsy.statoolinfos.core.Organization;
import fr.devinsy.statoolinfos.core.Organizations;
import fr.devinsy.statoolinfos.core.Service;
@ -126,6 +127,11 @@ public class StatAgent
for (Service service : organization.getServices())
{
result.stat(service);
for (Metrics metrics : service.getMetrics())
{
result.stat(metrics);
}
}
}
@ -155,6 +161,11 @@ public class StatAgent
for (Service service : organization.getServices())
{
result.stat(service);
for (Metrics metrics : service.getMetrics())
{
result.stat(metrics, organization);
}
}
}
@ -413,6 +424,11 @@ public class StatAgent
for (Service service : services)
{
result.stat(service);
for (Metrics metrics : service.getMetrics())
{
result.stat(metrics);
}
}
//

28
src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java
View file

@ -26,6 +26,7 @@ import org.apache.commons.lang3.StringUtils;
import fr.devinsy.statoolinfos.checker.PropertyChecker;
import fr.devinsy.statoolinfos.checker.PropertyChecks;
import fr.devinsy.statoolinfos.core.Metrics;
import fr.devinsy.statoolinfos.core.Organization;
import fr.devinsy.statoolinfos.core.Service;
import fr.devinsy.strings.StringList;
@ -265,6 +266,33 @@ public class PropertiesFileStats extends ArrayList<PropertiesFileStat>
return result;
}
/**
* Stat.
*
* @param metrics
* the metrics
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public void stat(final Metrics metrics, final Organization oranization) throws IOException
{
PropertiesFileStat stat = stat(metrics.getInputFile());
stat.setURL(metrics.getInputURL());
stat.setLocalName(metrics.getLocalFileName());
stat.setOrganization(oranization);
stat.setUpdateDate(metrics.getCrawledDate());
PropertyChecker checker = new PropertyChecker();
PropertyChecks checks = checker.checkMetrics(metrics.getInputFile());
stat.setAlertCount(checks.getAlertCount());
stat.setErrorCount(checks.getErrorCount());
stat.setVoidCount(checks.getVoidCount());
stat.setWarningCount(checks.getWarningCount());
add(stat);
}
/**
* Stat.
*