diff --git a/src/fr/devinsy/statoolinfos/checker/PropertyChecker.java b/src/fr/devinsy/statoolinfos/checker/PropertyChecker.java index df2ce4c..abbc117 100644 --- a/src/fr/devinsy/statoolinfos/checker/PropertyChecker.java +++ b/src/fr/devinsy/statoolinfos/checker/PropertyChecker.java @@ -57,9 +57,10 @@ public class PropertyChecker public static final String METRICS_DAYS = "^metrics\\.\\S+\\.\\d{4}\\.days$"; public static final String CRAWL = "^crawl\\.\\S+$"; - private PropertyRules serviceRules; private PropertyRules federationRules; private PropertyRules organizationRules; + private PropertyRules serviceRules; + private PropertyRules metricsRules; /** * Instantiates a new property checker. @@ -190,6 +191,23 @@ public class PropertyChecker this.serviceRules.add(METRICS_DAYS, DAYS, PropertyMode.OPTIONAL); this.serviceRules.add(CRAWL, ALL, PropertyMode.MANDATORY); + + // + this.metricsRules = new PropertyRules(); + + this.metricsRules.add("file.class", "^[Mm]etrics$", PropertyMode.MANDATORY); + this.metricsRules.add("file.generator", STRING, PropertyMode.WISHED); + this.metricsRules.add("file.datetime", DATETIME, PropertyMode.WISHED); + this.metricsRules.add("file.protocol", STRING, PropertyMode.WISHED); + + this.metricsRules.add(METRICS_NAME, STRING, PropertyMode.OPTIONAL); + this.metricsRules.add(METRICS_DESCRIPTION, STRING, PropertyMode.OPTIONAL); + this.metricsRules.add(METRICS_YEAR, NUMERIC, PropertyMode.OPTIONAL); + this.metricsRules.add(METRICS_MONTHS, MONTHS, PropertyMode.OPTIONAL); + this.metricsRules.add(METRICS_WEEKS, WEEKS, PropertyMode.OPTIONAL); + this.metricsRules.add(METRICS_DAYS, DAYS, PropertyMode.OPTIONAL); + + this.metricsRules.add(CRAWL, ALL, PropertyMode.MANDATORY); } /** @@ -365,6 +383,26 @@ public class PropertyChecker return result; } + /** + * Check metrics. + * + * @param file + * the file + * @return the property checks + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public PropertyChecks checkMetrics(final File file) throws IOException + { + PropertyChecks result; + + StringList lines = StringsUtils.load(file); + result = check(lines, this.metricsRules); + + // + return result; + } + /** * Check organization. * diff --git a/src/fr/devinsy/statoolinfos/core/Factory.java b/src/fr/devinsy/statoolinfos/core/Factory.java index 238d83b..dff5582 100644 --- a/src/fr/devinsy/statoolinfos/core/Factory.java +++ b/src/fr/devinsy/statoolinfos/core/Factory.java @@ -226,6 +226,49 @@ public class Factory return result; } + /** + * Load metrics. + * + * @param inputURL + * the input URL + * @param cache + * the cache + * @param prefixPath + * the prefix path + * @return the metrics + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public static Metrics loadMetrics(final URL inputURL, final CrawlCache cache, final String prefixPath) throws IOException + { + Metrics result; + + File inputFile = cache.restoreFile(inputURL); + + if (inputFile == null) + { + logger.warn("WARNING: metrics not found in cache [{}]", inputURL); + result = null; + } + else + { + PathProperties properties = PathPropertyUtils.load(inputFile); + result = new Metrics(properties); + result.setLocalFileNamePrefix(prefixPath); + result.setInputFile(inputFile); + result.setInputURL(inputURL); + + // + PropertyChecker checker = new PropertyChecker(); + PropertyChecks checks = checker.checkMetrics(result.getInputFile()); + result.getInputChecks().addAll(checks); + result.getInputChecks().setFileName(result.getLocalFileName()); + } + + // + return result; + } + /** * Load organization. * @@ -261,25 +304,19 @@ public class Factory // PropertyChecker checker = new PropertyChecker(); PropertyChecks checks = checker.checkOrganization(result.getInputFile()); + checks.setFileName(result.getLocalFileName()); result.getInputChecks().addAll(checks); - result.getInputChecks().setFileName(result.getLocalFileName()); + // PathProperties subs = result.getByPrefix("subs"); for (PathProperty property : subs) { if (StringUtils.startsWith(property.getValue(), "http")) { URL serviceInputURL = new URL(property.getValue()); - Service service = loadService(serviceInputURL, cache); + Service service = loadService(serviceInputURL, cache, result); if (service != null) { - service.setOrganization(result); - service.setLogoFileName(result.getTechnicalName() + "-" + service.getLogoFileName()); - - PropertyChecks subChecks = checker.checkService(service.getInputFile()); - service.getInputChecks().addAll(subChecks); - service.getInputChecks().setFileName(service.getLocalFileName()); - result.getServices().add(service); } } @@ -311,7 +348,7 @@ public class Factory * @return the service * @throws IOException */ - public static Service loadService(final URL inputURL, final CrawlCache cache) throws IOException + public static Service loadService(final URL inputURL, final CrawlCache cache, final Organization organization) throws IOException { Service result; @@ -326,9 +363,31 @@ public class Factory { PathProperties properties = PathPropertyUtils.load(inputFile); result = new Service(properties); + result.setOrganization(organization); result.setInputFile(inputFile); result.setInputURL(inputURL); - result.setLogoFileName(result.getTechnicalName() + "-logo" + StringUtils.defaultIfBlank(cache.getExtension(result.getLogoURL()), ".png")); + result.setLogoFileName(organization.getTechnicalName() + "-" + result.getTechnicalName() + "-logo" + StringUtils.defaultIfBlank(cache.getExtension(result.getLogoURL()), ".png")); + + // + PathProperties subs = result.getByPrefix("subs"); + for (PathProperty property : subs) + { + if (StringUtils.startsWith(property.getValue(), "http")) + { + URL metricsInputURL = new URL(property.getValue()); + Metrics metrics = loadMetrics(metricsInputURL, cache, organization.getTechnicalName() + "-" + result.getTechnicalName()); + if (metrics != null) + { + result.getMetrics().add(metrics); + } + } + } + + // + PropertyChecker checker = new PropertyChecker(); + PropertyChecks checks = checker.checkService(result.getInputFile()); + checks.setFileName(result.getLocalFileName()); + result.getInputChecks().addAll(checks); // CrawlJournal journal = cache.restoreJournal(); diff --git a/src/fr/devinsy/statoolinfos/core/Metrics.java b/src/fr/devinsy/statoolinfos/core/Metrics.java new file mode 100644 index 0000000..cb31b7e --- /dev/null +++ b/src/fr/devinsy/statoolinfos/core/Metrics.java @@ -0,0 +1,182 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.core; + +import java.io.File; +import java.net.URL; +import java.time.LocalDateTime; + +import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import fr.devinsy.statoolinfos.checker.PropertyChecks; +import fr.devinsy.statoolinfos.properties.PathProperties; +import fr.devinsy.statoolinfos.properties.PathPropertyList; + +/** + * The Class Metrics. + */ +public class Metrics extends PathPropertyList +{ + private static final long serialVersionUID = -1608084706095266037L; + + private static Logger logger = LoggerFactory.getLogger(Metrics.class); + + private String localFileNamePrefix; + private File inputFile; + private URL inputURL; + private PropertyChecks inputChecks; + + /** + * Instantiates a new service. + */ + public Metrics() + { + this(null); + } + + /** + * Instantiates a new service. + * + * @param properties + * the properties + */ + public Metrics(final PathProperties properties) + { + super(properties); + this.inputChecks = new PropertyChecks(); + } + + /** + * Gets the crawl date. + * + * @return the crawl date + */ + public LocalDateTime getCrawlDate() + { + LocalDateTime result; + + result = LocalDateTime.parse(get("crawl.datetime")); + + // + return result; + } + + /** + * Gets the crawled date. + * + * @return the crawled date + */ + public LocalDateTime getCrawledDate() + { + LocalDateTime result; + + result = LocalDateTime.parse(get("crawl.file.datetime")); + + if (result.getYear() == 1970) + { + result = getCrawlDate(); + } + + // + return result; + } + + public PropertyChecks getInputChecks() + { + return this.inputChecks; + } + + public File getInputFile() + { + return this.inputFile; + } + + public URL getInputURL() + { + return this.inputURL; + } + + /** + * Gets the local file base name. + * + * @return the local file base name + */ + public String getLocalFileBaseName() + { + String result; + + result = this.localFileNamePrefix + "-" + getTechnicalName(); + + // + return result; + } + + /** + * Gets the local name. + * + * @return the local name + */ + public String getLocalFileName() + { + String result; + + result = getLocalFileBaseName() + ".properties"; + + // + return result; + } + + public String getLocalFileNamePrefix() + { + return this.localFileNamePrefix; + } + + /** + * Gets the technical name. + * + * @return the technical name + */ + public String getTechnicalName() + { + String result; + + String name = FilenameUtils.getBaseName(this.inputURL.getPath()); + result = StatoolInfosUtils.toTechnicalName(name); + + // + return result; + } + + public void setInputFile(final File inputFile) + { + this.inputFile = inputFile; + } + + public void setInputURL(final URL inputURL) + { + this.inputURL = inputURL; + } + + public void setLocalFileNamePrefix(final String localFileNamePrefix) + { + this.localFileNamePrefix = localFileNamePrefix; + } +} diff --git a/src/fr/devinsy/statoolinfos/core/MetricsList.java b/src/fr/devinsy/statoolinfos/core/MetricsList.java new file mode 100644 index 0000000..f5a9826 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/core/MetricsList.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2021 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.core; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * The Class MetricsList. + */ +public class MetricsList extends ArrayList +{ + private static final long serialVersionUID = 6241734209264875306L; + + /** + * Instantiates a new metrics list. + */ + public MetricsList() + { + super(); + } + + /** + * Reverse. + * + * @return the services + */ + public MetricsList reverse() + { + MetricsList result; + + Collections.reverse(this); + + result = this; + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/core/Organization.java b/src/fr/devinsy/statoolinfos/core/Organization.java index 8d3703d..8075069 100644 --- a/src/fr/devinsy/statoolinfos/core/Organization.java +++ b/src/fr/devinsy/statoolinfos/core/Organization.java @@ -253,7 +253,7 @@ public class Organization extends PathPropertyList result.addAll(this.inputChecks); for (Service service : this.services) { - result.addAll(service.getInputChecks()); + result.addAll(service.getInputChecksAll()); } // @@ -295,6 +295,21 @@ public class Organization extends PathPropertyList return result; } + /** + * Gets the local file base name. + * + * @return the local file base name + */ + public String getLocalFileBaseName() + { + String result; + + result = getTechnicalName(); + + // + return result; + } + /** * Gets the local file name. * @@ -304,7 +319,7 @@ public class Organization extends PathPropertyList { String result; - result = getTechnicalName() + ".properties"; + result = getLocalFileBaseName() + ".properties"; // return result; diff --git a/src/fr/devinsy/statoolinfos/core/PropertyClassType.java b/src/fr/devinsy/statoolinfos/core/PropertyClassType.java index c6020ad..0fa48b4 100644 --- a/src/fr/devinsy/statoolinfos/core/PropertyClassType.java +++ b/src/fr/devinsy/statoolinfos/core/PropertyClassType.java @@ -71,7 +71,14 @@ public enum PropertyClassType } break; case METRICS: - result = true; + if (parent == METRICS) + { + result = false; + } + else + { + result = true; + } break; default: result = false; diff --git a/src/fr/devinsy/statoolinfos/core/Service.java b/src/fr/devinsy/statoolinfos/core/Service.java index 4372545..f80075a 100644 --- a/src/fr/devinsy/statoolinfos/core/Service.java +++ b/src/fr/devinsy/statoolinfos/core/Service.java @@ -104,6 +104,7 @@ public class Service extends PathPropertyList private File inputFile; private URL inputURL; private String logoFileName; + private MetricsList metricsList; private PropertyChecks inputChecks; private CrawlJournal crawlJournal; @@ -126,6 +127,7 @@ public class Service extends PathPropertyList super(properties); this.inputChecks = new PropertyChecks(); this.crawlJournal = new CrawlJournal(); + this.metricsList = new MetricsList(); } /** @@ -329,6 +331,28 @@ public class Service extends PathPropertyList return this.inputChecks; } + /** + * Gets the input checks all. + * + * @return the input checks all + */ + public PropertyChecks getInputChecksAll() + { + PropertyChecks result; + + result = new PropertyChecks(); + + result.addAll(this.inputChecks); + + for (Metrics metrics : getMetrics()) + { + result.addAll(metrics.getInputChecks()); + } + + // + return result; + } + public File getInputFile() { return this.inputFile; @@ -369,6 +393,21 @@ public class Service extends PathPropertyList return result; } + /** + * Gets the local name. + * + * @return the local name + */ + public String getLocalFileBaseName() + { + String result; + + result = this.organization.getTechnicalName() + "-" + getTechnicalName(); + + // + return result; + } + /** * Gets the local name. * @@ -378,7 +417,7 @@ public class Service extends PathPropertyList { String result; - result = this.organization.getTechnicalName() + "-" + getTechnicalName() + ".properties"; + result = getLocalFileBaseName() + ".properties"; // return result; @@ -451,6 +490,11 @@ public class Service extends PathPropertyList return result; } + public MetricsList getMetrics() + { + return this.metricsList; + } + /** * Gets the metric years. * @@ -509,21 +553,6 @@ public class Service extends PathPropertyList return this.organization; } - /** - * Gets the property file name. - * - * @return the property file name - */ - public String getPropertyFileName() - { - String result; - - result = getOrganization().getTechnicalName() + "-" + getTechnicalName() + ".properties"; - - // - return result; - } - /** * Gets the registration load. * diff --git a/src/fr/devinsy/statoolinfos/htmlize/PropertyFileCheckPage.java b/src/fr/devinsy/statoolinfos/htmlize/PropertyFileCheckPage.java index 1623a53..3a6212e 100644 --- a/src/fr/devinsy/statoolinfos/htmlize/PropertyFileCheckPage.java +++ b/src/fr/devinsy/statoolinfos/htmlize/PropertyFileCheckPage.java @@ -30,6 +30,7 @@ import fr.devinsy.statoolinfos.HtmlizerContext; import fr.devinsy.statoolinfos.checker.PropertyCheck; import fr.devinsy.statoolinfos.checker.PropertyChecks; import fr.devinsy.statoolinfos.core.Federation; +import fr.devinsy.statoolinfos.core.Metrics; import fr.devinsy.statoolinfos.core.Organization; import fr.devinsy.statoolinfos.core.Service; import fr.devinsy.statoolinfos.core.Service.Status; @@ -75,26 +76,35 @@ public class PropertyFileCheckPage { checks = organization.getInputChecks(); page = htmlize("Organisation", checks); - FileUtils.write(new File(htmlizeDirectory, organization.getTechnicalName() + "-check.xhtml"), page, StandardCharsets.UTF_8); + FileUtils.write(new File(htmlizeDirectory, organization.getLocalFileBaseName() + "-check.xhtml"), page, StandardCharsets.UTF_8); + // Ignore ghost organizations. if (organization.getServiceCount() > 0) { alerts = checks.getAlertLines().setFileName(organization.getLocalFileName()); allAlertChecks.addAll(alerts); organizationAlertChecks.addAll(alerts); - } - for (Service service : organization.getServices()) - { - checks = service.getInputChecks(); - page = htmlize("Service", checks); - FileUtils.write(new File(htmlizeDirectory, organization.getTechnicalName() + "-" + service.getTechnicalName() + "-check.xhtml"), page, StandardCharsets.UTF_8); - - if (organization.getServiceCount() > 0) + for (Service service : organization.getServices()) { + checks = service.getInputChecksAll(); + page = htmlize("Service", checks); + FileUtils.write(new File(htmlizeDirectory, service.getLocalFileBaseName() + "-check.xhtml"), page, StandardCharsets.UTF_8); + alerts = checks.getAlertLines().setFileName(service.getLocalFileName()); allAlertChecks.addAll(alerts); serviceAlertChecks.addAll(alerts); + + for (Metrics metrics : service.getMetrics()) + { + checks = metrics.getInputChecks(); + page = htmlize("Metrics", checks); + FileUtils.write(new File(htmlizeDirectory, metrics.getLocalFileBaseName() + "-check.xhtml"), page, StandardCharsets.UTF_8); + + alerts = checks.getAlertLines().setFileName(metrics.getLocalFileName()); + allAlertChecks.addAll(alerts); + serviceAlertChecks.addAll(alerts); + } } } } diff --git a/src/fr/devinsy/statoolinfos/htmlize/ServicePage.java b/src/fr/devinsy/statoolinfos/htmlize/ServicePage.java index df46d50..b1620c6 100644 --- a/src/fr/devinsy/statoolinfos/htmlize/ServicePage.java +++ b/src/fr/devinsy/statoolinfos/htmlize/ServicePage.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; import fr.devinsy.catgenerator.core.BirdGenerator; import fr.devinsy.statoolinfos.HtmlizerContext; import fr.devinsy.statoolinfos.checker.PropertyChecks; +import fr.devinsy.statoolinfos.core.Metrics; import fr.devinsy.statoolinfos.core.Organization; import fr.devinsy.statoolinfos.core.Service; import fr.devinsy.statoolinfos.core.StatoolInfosException; @@ -67,12 +68,19 @@ public class ServicePage // logger.info("Htmlize service properties file: {}.", service.getName()); FileUtils.copyFile(service.getInputFile(), - new File(htmlizeDirectory, service.getOrganization().getTechnicalName() + "-" + service.getTechnicalName() + ".properties")); + new File(htmlizeDirectory, service.getLocalFileBaseName() + ".properties")); + + // + for (Metrics metrics : service.getMetrics()) + { + FileUtils.copyFile(metrics.getInputFile(), + new File(htmlizeDirectory, metrics.getLocalFileBaseName() + ".properties")); + } // logger.info("Htmlize service page: {}.", service.getName()); String page = ServicePage.htmlize(service.getOrganization(), service); - FileUtils.write(new File(htmlizeDirectory, service.getOrganization().getTechnicalName() + "-" + service.getTechnicalName() + ".xhtml"), page, StandardCharsets.UTF_8); + FileUtils.write(new File(htmlizeDirectory, service.getLocalFileBaseName() + ".xhtml"), page, StandardCharsets.UTF_8); } /** @@ -232,7 +240,7 @@ public class ServicePage } { - PropertyChecks checks = service.getInputChecks(); + PropertyChecks checks = service.getInputChecksAll(); data.setContent("errorCount", checks.getErrorCount()); data.setContent("warningCount", checks.getWarningCount()); data.setContent("voidCount", checks.getVoidCount()); diff --git a/src/fr/devinsy/statoolinfos/htmlize/propertyFileCheck.xhtml b/src/fr/devinsy/statoolinfos/htmlize/propertyFileCheck.xhtml index b8e46bd..41c0c8e 100644 --- a/src/fr/devinsy/statoolinfos/htmlize/propertyFileCheck.xhtml +++ b/src/fr/devinsy/statoolinfos/htmlize/propertyFileCheck.xhtml @@ -31,7 +31,7 @@ 000 - n/a + n/a n/a @@ -42,7 +42,7 @@ 000 - n/a + n/a n/a @@ -53,7 +53,7 @@ 000 - n/a + n/a n/a diff --git a/src/fr/devinsy/statoolinfos/stats/StatAgent.java b/src/fr/devinsy/statoolinfos/stats/StatAgent.java index f2b828c..f3dbbfb 100644 --- a/src/fr/devinsy/statoolinfos/stats/StatAgent.java +++ b/src/fr/devinsy/statoolinfos/stats/StatAgent.java @@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory; import fr.devinsy.statoolinfos.core.Categories; import fr.devinsy.statoolinfos.core.Category; import fr.devinsy.statoolinfos.core.Federation; +import fr.devinsy.statoolinfos.core.Metrics; import fr.devinsy.statoolinfos.core.Organization; import fr.devinsy.statoolinfos.core.Organizations; import fr.devinsy.statoolinfos.core.Service; @@ -126,6 +127,11 @@ public class StatAgent for (Service service : organization.getServices()) { result.stat(service); + + for (Metrics metrics : service.getMetrics()) + { + result.stat(metrics); + } } } @@ -155,6 +161,11 @@ public class StatAgent for (Service service : organization.getServices()) { result.stat(service); + + for (Metrics metrics : service.getMetrics()) + { + result.stat(metrics, organization); + } } } @@ -413,6 +424,11 @@ public class StatAgent for (Service service : services) { result.stat(service); + + for (Metrics metrics : service.getMetrics()) + { + result.stat(metrics); + } } // diff --git a/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java b/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java index 4483de3..3a3852f 100644 --- a/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java +++ b/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java @@ -26,6 +26,7 @@ import org.apache.commons.lang3.StringUtils; import fr.devinsy.statoolinfos.checker.PropertyChecker; import fr.devinsy.statoolinfos.checker.PropertyChecks; +import fr.devinsy.statoolinfos.core.Metrics; import fr.devinsy.statoolinfos.core.Organization; import fr.devinsy.statoolinfos.core.Service; import fr.devinsy.strings.StringList; @@ -265,6 +266,33 @@ public class PropertiesFileStats extends ArrayList return result; } + /** + * Stat. + * + * @param metrics + * the metrics + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public void stat(final Metrics metrics, final Organization oranization) throws IOException + { + PropertiesFileStat stat = stat(metrics.getInputFile()); + + stat.setURL(metrics.getInputURL()); + stat.setLocalName(metrics.getLocalFileName()); + stat.setOrganization(oranization); + stat.setUpdateDate(metrics.getCrawledDate()); + + PropertyChecker checker = new PropertyChecker(); + PropertyChecks checks = checker.checkMetrics(metrics.getInputFile()); + stat.setAlertCount(checks.getAlertCount()); + stat.setErrorCount(checks.getErrorCount()); + stat.setVoidCount(checks.getVoidCount()); + stat.setWarningCount(checks.getWarningCount()); + + add(stat); + } + /** * Stat. *