From da17e52d37882de364f576dc99eca26beaaa9efc Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Wed, 23 Sep 2020 21:56:40 +0200 Subject: [PATCH] Added property files statistics. --- .../statoolinfos/core/Configuration.java | 2 +- src/fr/devinsy/statoolinfos/core/Factory.java | 60 +++- .../devinsy/statoolinfos/core/Federation.java | 15 +- .../statoolinfos/core/Organization.java | 31 +- src/fr/devinsy/statoolinfos/core/Service.java | 21 +- .../statoolinfos/core/StatoolInfosUtils.java | 62 ++++ .../statoolinfos/crawl/CrawlCache.java | 115 +++++++- .../devinsy/statoolinfos/crawl/Crawler.java | 35 ++- .../statoolinfos/htmlize/Htmlizer.java | 21 +- .../htmlize/PropertiesFilesPage.java | 40 ++- .../htmlize/PropertyStatsPage.java | 4 +- .../htmlize/propertiesFiles.xhtml | 25 +- .../properties/PathProperties.java | 10 + .../properties/PathPropertyList.java | 6 + .../stats/PropertiesFileStat.java | 38 --- .../stats/PropertiesFileStats.java | 37 --- .../devinsy/statoolinfos/stats/StatAgent.java | 35 +++ .../stats/{ => properties}/PropertyStat.java | 2 +- .../PropertyStatComparator.java | 2 +- .../{ => properties}/PropertyStatList.java | 2 +- .../{ => properties}/PropertyStatSet.java | 2 +- .../stats/{ => properties}/PropertyStats.java | 2 +- .../propertyfiles/PropertiesFileStat.java | 150 ++++++++++ .../PropertiesFileStatComparator.java | 271 ++++++++++++++++++ .../propertyfiles/PropertiesFileStats.java | 171 +++++++++++ 25 files changed, 985 insertions(+), 174 deletions(-) delete mode 100644 src/fr/devinsy/statoolinfos/stats/PropertiesFileStat.java delete mode 100644 src/fr/devinsy/statoolinfos/stats/PropertiesFileStats.java rename src/fr/devinsy/statoolinfos/stats/{ => properties}/PropertyStat.java (97%) rename src/fr/devinsy/statoolinfos/stats/{ => properties}/PropertyStatComparator.java (98%) rename src/fr/devinsy/statoolinfos/stats/{ => properties}/PropertyStatList.java (98%) rename src/fr/devinsy/statoolinfos/stats/{ => properties}/PropertyStatSet.java (96%) rename src/fr/devinsy/statoolinfos/stats/{ => properties}/PropertyStats.java (98%) create mode 100644 src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStat.java create mode 100644 src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStatComparator.java create mode 100644 src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java diff --git a/src/fr/devinsy/statoolinfos/core/Configuration.java b/src/fr/devinsy/statoolinfos/core/Configuration.java index 21a32c9..7e3b678 100644 --- a/src/fr/devinsy/statoolinfos/core/Configuration.java +++ b/src/fr/devinsy/statoolinfos/core/Configuration.java @@ -189,7 +189,7 @@ public class Configuration extends PathPropertyList * * @return the crawl input */ - public File getCrawlInput() + public File getCrawlInputFile() { File result; diff --git a/src/fr/devinsy/statoolinfos/core/Factory.java b/src/fr/devinsy/statoolinfos/core/Factory.java index 97ab525..db150c1 100644 --- a/src/fr/devinsy/statoolinfos/core/Factory.java +++ b/src/fr/devinsy/statoolinfos/core/Factory.java @@ -84,16 +84,15 @@ public class Factory PathProperties properties = PathPropertyUtils.load(federationFile); result = new Federation(properties); - result.setLocalFile(federationFile); + result.setInputFile(federationFile); PathProperties subs = result.getByPrefix("subs"); for (PathProperty property : subs) { if (StringUtils.startsWith(property.getValue(), "http")) { - File subFile = cache.restoreFile(new URL(property.getValue())); - Organization organization = loadOrganization(subFile, cache); - organization.setFederation(result); + URL inputURL = new URL(property.getValue()); + Organization organization = loadOrganization(inputURL, cache); result.getOrganizations().add(organization); } } @@ -119,15 +118,53 @@ public class Factory PathProperties properties = PathPropertyUtils.load(organizationFile); result = new Organization(properties); - result.setLocalFile(organizationFile); + result.setInputFile(organizationFile); PathProperties subs = result.getByPrefix("subs"); for (PathProperty property : subs) { if (StringUtils.startsWith(property.getValue(), "http")) { - File subFile = cache.restoreFile(new URL(property.getValue())); - Service service = loadService(subFile); + URL serviceInputFile = new URL(property.getValue()); + Service service = loadService(serviceInputFile, cache); + service.setOrganization(result); + result.getServices().add(service); + } + } + + // + return result; + } + + /** + * Load organization. + * + * @param inputURL + * the input + * @param cache + * the cache + * @return the organization + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public static Organization loadOrganization(final URL inputURL, final CrawlCache cache) throws IOException + { + Organization result; + + File inputFile = cache.restoreFile(inputURL); + + PathProperties properties = PathPropertyUtils.load(inputFile); + result = new Organization(properties); + result.setInputFile(inputFile); + result.setInputURL(inputURL); + + PathProperties subs = result.getByPrefix("subs"); + for (PathProperty property : subs) + { + if (StringUtils.startsWith(property.getValue(), "http")) + { + URL serviceInputURL = new URL(property.getValue()); + Service service = loadService(serviceInputURL, cache); service.setOrganization(result); result.getServices().add(service); } @@ -143,13 +180,16 @@ public class Factory * @return the service * @throws IOException */ - public static Service loadService(final File serviceFile) throws IOException + public static Service loadService(final URL inputURL, final CrawlCache cache) throws IOException { Service result; - PathProperties properties = PathPropertyUtils.load(serviceFile); + File inputFile = cache.restoreFile(inputURL); + + PathProperties properties = PathPropertyUtils.load(inputFile); result = new Service(properties); - result.setLocalFile(serviceFile); + result.setInputFile(inputFile); + result.setInputURL(inputURL); // return result; diff --git a/src/fr/devinsy/statoolinfos/core/Federation.java b/src/fr/devinsy/statoolinfos/core/Federation.java index 01ae2f4..be1d945 100644 --- a/src/fr/devinsy/statoolinfos/core/Federation.java +++ b/src/fr/devinsy/statoolinfos/core/Federation.java @@ -34,7 +34,7 @@ public class Federation extends PathPropertyList { private static final long serialVersionUID = -8970835291634661580L; private Organizations organizations; - private File localFile; + private File inputFile; /** * Instantiates a new federation. @@ -100,14 +100,9 @@ public class Federation extends PathPropertyList return result; } - /** - * Gets the local file. - * - * @return the local file - */ - public File getLocalFile() + public File getInputFile() { - return this.localFile; + return this.inputFile; } /** @@ -203,8 +198,8 @@ public class Federation extends PathPropertyList return result; } - public void setLocalFile(final File localFile) + public void setInputFile(final File inputFile) { - this.localFile = localFile; + this.inputFile = inputFile; } } diff --git a/src/fr/devinsy/statoolinfos/core/Organization.java b/src/fr/devinsy/statoolinfos/core/Organization.java index d311cab..508a374 100644 --- a/src/fr/devinsy/statoolinfos/core/Organization.java +++ b/src/fr/devinsy/statoolinfos/core/Organization.java @@ -35,7 +35,8 @@ public class Organization extends PathPropertyList private static final long serialVersionUID = -2709210934548224213L; private Federation federation; private Services services; - private File localFile; + private File inputFile; + private URL inputURL; /** * Instantiates a new organization. @@ -81,9 +82,24 @@ public class Organization extends PathPropertyList return this.federation; } - public File getLocalFile() + public File getInputFile() { - return this.localFile; + return this.inputFile; + } + + /** + * Gets the crawled input URL. + * + * @return the crawled input URL + */ + public URL getInputURL() + { + URL result; + + result = this.inputURL; + + // + return result; } /** @@ -181,8 +197,13 @@ public class Organization extends PathPropertyList this.federation = federation; } - public void setLocalFile(final File localFile) + public void setInputFile(final File inputFile) { - this.localFile = localFile; + this.inputFile = inputFile; + } + + public void setInputURL(final URL inputURL) + { + this.inputURL = inputURL; } } diff --git a/src/fr/devinsy/statoolinfos/core/Service.java b/src/fr/devinsy/statoolinfos/core/Service.java index 1cfc8be..e746bc2 100644 --- a/src/fr/devinsy/statoolinfos/core/Service.java +++ b/src/fr/devinsy/statoolinfos/core/Service.java @@ -34,7 +34,8 @@ public class Service extends PathPropertyList { private static final long serialVersionUID = 3629841771102288863L; private Organization organization; - private File localFile; + private File inputFile; + private URL inputURL; /** * Instantiates a new service. @@ -70,9 +71,14 @@ public class Service extends PathPropertyList return result; } - public File getLocalFile() + public File getInputFile() { - return this.localFile; + return this.inputFile; + } + + public URL getInputURL() + { + return this.inputURL; } /** @@ -160,9 +166,14 @@ public class Service extends PathPropertyList return result; } - public void setLocalFile(final File localFile) + public void setInputFile(final File inputFile) { - this.localFile = localFile; + this.inputFile = inputFile; + } + + public void setInputURL(final URL inputURL) + { + this.inputURL = inputURL; } public void setOrganization(final Organization organization) diff --git a/src/fr/devinsy/statoolinfos/core/StatoolInfosUtils.java b/src/fr/devinsy/statoolinfos/core/StatoolInfosUtils.java index e626900..c49a2f6 100644 --- a/src/fr/devinsy/statoolinfos/core/StatoolInfosUtils.java +++ b/src/fr/devinsy/statoolinfos/core/StatoolInfosUtils.java @@ -20,9 +20,12 @@ package fr.devinsy.statoolinfos.core; import java.io.File; import java.io.IOException; +import java.net.HttpURLConnection; import java.net.URL; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.Iterator; @@ -102,6 +105,42 @@ public class StatoolInfosUtils FileUtils.copyURLToFile(url, finalTarget); } + /** + * Epoch to local date time. + * + * @param epoch + * the epoch + * @return the local date time + */ + public static LocalDateTime epochToLocalDateTime(final long epoch) + { + LocalDateTime result; + + result = Instant.ofEpochMilli(epoch).atZone(ZoneId.systemDefault()).toLocalDateTime(); + + // + return result; + } + + /** + * File local date time. + * + * @param file + * the file + * @return the local date time + */ + public static LocalDateTime fileLocalDateTime(final File file) + { + LocalDateTime result; + + long epoch = file.lastModified(); + + result = epochToLocalDateTime(epoch); + + // + return result; + } + /** * Generate cat logo. * @@ -417,4 +456,27 @@ public class StatoolInfosUtils { } } + + /** + * Url last modified. + * + * @param url + * the url + * @return the local date time + * @throws IOException + */ + public static LocalDateTime urlLastModified(final URL url) throws IOException + { + LocalDateTime result; + + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + Long epoch = connection.getLastModified(); + connection.disconnect(); + result = epochToLocalDateTime(epoch); + // result = + // Instant.ofEpochMilli(epoch).atZone(ZoneId.of("GMT")).toLocalDateTime(); + + // + return result; + } } diff --git a/src/fr/devinsy/statoolinfos/crawl/CrawlCache.java b/src/fr/devinsy/statoolinfos/crawl/CrawlCache.java index 92f1ecc..e16714c 100644 --- a/src/fr/devinsy/statoolinfos/crawl/CrawlCache.java +++ b/src/fr/devinsy/statoolinfos/crawl/CrawlCache.java @@ -107,21 +107,21 @@ public class CrawlCache /** * Restore file. * - * @param url - * the url + * @param key + * the key * @return the file */ - public File restoreFile(final URL url) + public File restoreFile(final String key) { File result; - if (url == null) + if (StringUtils.isBlank(key)) { throw new IllegalArgumentException("Null parameter."); } else { - result = buildFile(url.toString()); + result = buildFile(key); if (!result.exists()) { result = null; @@ -132,6 +132,42 @@ public class CrawlCache return result; } + /** + * Restore file. + * + * @param url + * the url + * @return the file + */ + public File restoreFile(final URL url) + { + File result; + + result = restoreFile(url.toString()); + + // + return result; + } + + /** + * Restore file to. + * + * @param key + * the key + * @param target + * the target + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public void restoreFileTo(final String key, final File target) throws IOException + { + File logoFile = restoreFile(key); + if (logoFile != null) + { + FileUtils.copyFile(logoFile, target); + } + } + /** * Restore file to. * @@ -144,14 +180,7 @@ public class CrawlCache */ public void restoreFileTo(final URL url, final File target) throws IOException { - if (url != null) - { - File logoFile = restoreFile(url); - if (logoFile != null) - { - FileUtils.copyFile(logoFile, target); - } - } + restoreFile(url.toString()); } /** @@ -238,6 +267,33 @@ public class CrawlCache return result; } + /** + * Store. + * + * @param key + * the key + * @return the file + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public File store(final String key, final File source) throws IOException + { + File result; + + if (StringUtils.isBlank(key)) + { + result = null; + } + else + { + result = buildFile(key); + FileUtils.copyFile(source, result); + } + + // + return result; + } + /** * Store. * @@ -265,6 +321,36 @@ public class CrawlCache return result; } + /** + * Store properties. + * + * @param key + * the key + * @param properties + * the properties + * @return the file + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public File storeProperties(final String key, final PathProperties properties) throws IOException + { + File result; + + if (StringUtils.isBlank(key)) + { + result = null; + } + else + { + result = buildFile(key); + + PathPropertyUtils.save(result, properties); + } + + // + return result; + } + /** * Store. * @@ -286,8 +372,7 @@ public class CrawlCache } else { - String key = url.toString() + ".properties"; - result = buildFile(key); + result = buildFile(url.toString()); PathPropertyUtils.save(result, properties); } diff --git a/src/fr/devinsy/statoolinfos/crawl/Crawler.java b/src/fr/devinsy/statoolinfos/crawl/Crawler.java index 3be9680..6790e67 100644 --- a/src/fr/devinsy/statoolinfos/crawl/Crawler.java +++ b/src/fr/devinsy/statoolinfos/crawl/Crawler.java @@ -21,8 +21,11 @@ package fr.devinsy.statoolinfos.crawl; import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +33,7 @@ import org.slf4j.LoggerFactory; import fr.devinsy.statoolinfos.core.Configuration; import fr.devinsy.statoolinfos.core.Factory; import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathProperty; import fr.devinsy.statoolinfos.properties.PathPropertyList; @@ -89,10 +93,18 @@ public class Crawler CrawlCache cache = configuration.getCrawlCache(); - PathProperties input = PathPropertyUtils.load(configuration.getCrawlInput()); + PathProperties input = PathPropertyUtils.load(configuration.getCrawlInputFile()); - cache.storeQuietly(input.getURL("federation.logo")); - cache.storeQuietly(input.getURL("organization.logo")); + if (configuration.isFederation()) + { + cache.store(input.get("federation.name"), configuration.getCrawlInputFile()); + cache.storeQuietly(input.getURL("federation.logo")); + } + else + { + cache.store(input.get("organization.name"), configuration.getCrawlInputFile()); + cache.storeQuietly(input.getURL("organization.logo")); + } PathProperties subs = input.getByPrefix("subs"); for (PathProperty property : subs) @@ -137,20 +149,29 @@ public class Crawler { logger.info("Crawling " + url); + // Crawl. File file = cache.store(url); - PathProperties properties = PathPropertyUtils.load(file); + // Build crawl data. PathProperties crawlSection = new PathPropertyList(); crawlSection.put("crawl.crawler", "StatoolInfos"); crawlSection.put("crawl.datetime", LocalDateTime.now().toString()); crawlSection.put("crawl.url", url.toString()); - properties.add(crawlSection); - cache.storeProperties(url, properties); + crawlSection.put("crawl.file.size", FileUtils.sizeOf(file)); + crawlSection.put("crawl.file.datetime", StatoolInfosUtils.urlLastModified(url).toString()); + crawlSection.put("crawl.file.sha1", DigestUtils.sha1Hex(FileUtils.readFileToByteArray(file))); + + // Add crawl data in crawled file. + String lines = crawlSection.toStringListFormatted().toStringSeparatedBy('\n'); + FileUtils.write(file, FileUtils.readFileToString(file, StandardCharsets.UTF_8) + "\n" + lines, StandardCharsets.UTF_8); + + // Crawl another resources. + PathProperties properties = PathPropertyUtils.load(file); cache.storeQuietly(properties.getURL("organization.logo")); cache.storeQuietly(properties.getURL("service.logo")); - // + // Crawl subs. PathProperties subs = properties.getByPrefix("subs"); for (PathProperty property : subs) { diff --git a/src/fr/devinsy/statoolinfos/htmlize/Htmlizer.java b/src/fr/devinsy/statoolinfos/htmlize/Htmlizer.java index ee8d0bd..3f03f76 100644 --- a/src/fr/devinsy/statoolinfos/htmlize/Htmlizer.java +++ b/src/fr/devinsy/statoolinfos/htmlize/Htmlizer.java @@ -35,8 +35,9 @@ import fr.devinsy.statoolinfos.core.Service; import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.crawl.CrawlCache; -import fr.devinsy.statoolinfos.stats.PropertyStats; import fr.devinsy.statoolinfos.stats.StatAgent; +import fr.devinsy.statoolinfos.stats.properties.PropertyStats; +import fr.devinsy.statoolinfos.stats.propertyfiles.PropertiesFileStats; /** * The Class Htmlizer. @@ -212,8 +213,8 @@ public class Htmlizer // Manage the logo file. logger.info("Htmlize federation logo."); cache.restoreLogoTo(federation.getLogoURL(), new File(htmlizeDirectory, federation.getTechnicalName() + "-logo.png"), federation.getTechnicalName()); - logger.info("Htmlize federation properties file."); - FileUtils.copyFile(federation.getLocalFile(), new File(htmlizeDirectory, federation.getTechnicalName() + ".properties")); + logger.info("Htmlize federation properties files."); + FileUtils.copyFile(federation.getInputFile(), new File(htmlizeDirectory, federation.getTechnicalName() + ".properties")); // logger.info("Htmlize about page."); @@ -231,7 +232,7 @@ public class Htmlizer logger.info("Htmlize organization logo: {}.", organization.getName()); cache.restoreLogoTo(organization.getLogoURL(), new File(htmlizeDirectory, organization.getTechnicalName() + "-logo.png"), organization.getTechnicalName()); logger.info("Htmlize organization properties file: {}.", organization.getName()); - FileUtils.copyFile(organization.getLocalFile(), new File(htmlizeDirectory, organization.getTechnicalName() + ".properties")); + FileUtils.copyFile(organization.getInputFile(), new File(htmlizeDirectory, organization.getTechnicalName() + ".properties")); // logger.info("Htmlize organization page: {}.", organization.getName()); @@ -244,7 +245,7 @@ public class Htmlizer logger.info("Htmlize service logo: {}.", service.getName()); cache.restoreLogoTo(service.getLogoURL(), new File(htmlizeDirectory, organization.getTechnicalName() + "-" + service.getTechnicalName() + "-logo.png"), service.getTechnicalName()); logger.info("Htmlize service properties file: {}.", service.getName()); - FileUtils.copyFile(service.getLocalFile(), + FileUtils.copyFile(service.getInputFile(), new File(htmlizeDirectory, organization.getTechnicalName() + "-" + service.getTechnicalName() + ".properties")); logger.info("Htmlize service page: {}.", service.getName()); @@ -258,10 +259,12 @@ public class Htmlizer page = ServicesPage.build(federation.getAllServices()); FileUtils.write(new File(htmlizeDirectory, "services.xhtml"), page, StandardCharsets.UTF_8); - logger.info("Htmlize propertiesFiles page."); - page = PropertiesFilesPage.build(federation); - FileUtils.write(new File(htmlizeDirectory, "propertiesFiles.xhtml"), page, StandardCharsets.UTF_8); - + { + logger.info("Htmlize propertiesFiles page."); + PropertiesFileStats stats = StatAgent.statAllPropertiesFiles(federation, cache).sortByName(); + page = PropertiesFilesPage.build(stats); + FileUtils.write(new File(htmlizeDirectory, "propertiesFiles.xhtml"), page, StandardCharsets.UTF_8); + } // { logger.info("Htmlize propertyStats page."); diff --git a/src/fr/devinsy/statoolinfos/htmlize/PropertiesFilesPage.java b/src/fr/devinsy/statoolinfos/htmlize/PropertiesFilesPage.java index 3751ed5..847f030 100644 --- a/src/fr/devinsy/statoolinfos/htmlize/PropertiesFilesPage.java +++ b/src/fr/devinsy/statoolinfos/htmlize/PropertiesFilesPage.java @@ -25,9 +25,9 @@ import java.util.Locale; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import fr.devinsy.statoolinfos.core.Federation; -import fr.devinsy.statoolinfos.core.Organization; import fr.devinsy.statoolinfos.core.StatoolInfosException; +import fr.devinsy.statoolinfos.stats.propertyfiles.PropertiesFileStat; +import fr.devinsy.statoolinfos.stats.propertyfiles.PropertiesFileStats; import fr.devinsy.statoolinfos.util.BuildInformation; import fr.devinsy.xidyn.XidynException; import fr.devinsy.xidyn.data.TagDataManager; @@ -49,41 +49,37 @@ public class PropertiesFilesPage * @throws StatoolInfosException * the statool infos exception */ - public static String build(final Federation federation) throws StatoolInfosException + public static String build(final PropertiesFileStats stats) throws StatoolInfosException { String result; try { - logger.debug("Building federation page {}…", federation.getName()); + logger.debug("Building propertyFiles page."); TagDataManager data = new TagDataManager(); data.setContent("versionsup", BuildInformation.instance().version()); data.setContent("lastUpdateDate", LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd/MM/yyyy HH':'mm", Locale.FRANCE))); - data.setAttribute("federationRawButton", "href", federation.getTechnicalName() + ".properties"); - - data.setAttribute("federationLogo", "src", federation.getTechnicalName() + "-logo.png"); - data.setEscapedContent("federationName", federation.getName()); - data.setEscapedContent("federationDescription", federation.getDescription()); - data.setContent("organizationCount", federation.getOrganizations().size()); - data.setContent("serviceCount", federation.getServiceCount()); + data.setContent("fileCount", stats.size()); // int index = 0; - - // - - // - for (Organization organization : federation.getOrganizations()) + for (PropertiesFileStat stat : stats) { - data.setAttribute("organizationListLine", index, "organizationListLineNameLink", "href", organization.getTechnicalName() + ".xhtml"); - data.setAttribute("organizationListLine", index, "organizationListLineLogo", "src", organization.getTechnicalName() + "-logo.png"); - data.setEscapedContent("organizationListLine", index, "organizationListLineNameValue", organization.getName()); - data.setEscapedContent("organizationListLine", index, "organizationListLineUrlLink", organization.getWebsite()); - data.setAttribute("organizationListLine", index, "organizationListLineUrlLink", "href", organization.getWebsite()); - data.setContent("organizationListLine", index, "organizationListLineServiceCount", organization.getServiceCount()); + data.setAttribute("fileListLine", index, "fileListLineNameLink", "href", stat.getLocalName() + ".xhtml"); + data.setEscapedContent("fileListLine", index, "fileListLineNameLink", stat.getLocalName()); + + data.setAttribute("fileListLine", index, "fileListLineOwnerLink", "href", stat.getOrganization().getTechnicalName() + ".xhtml"); + data.setEscapedContent("fileListLine", index, "fileListLineNameValue", stat.getOrganization().getTechnicalName()); + data.setAttribute("fileListLine", index, "fileListLineOwnerLogo", "src", stat.getOrganization().getTechnicalName() + "-logo.png"); + + data.setContent("fileListLine", index, "fileListLineLineCount", stat.getLineCount()); + data.setContent("fileListLine", index, "fileListLineActiveCount", stat.getActiveLineCount()); + data.setContent("fileListLine", index, "fileListLineBlankPropertyCount", stat.getBlankPropertyCount()); + data.setContent("fileListLine", index, "fileListLineFilledPropertyCount", stat.getFilledPropertyCount()); + data.setContent("fileListLine", index, "fileListLineErrorCount", stat.getErrorCount()); index += 1; } diff --git a/src/fr/devinsy/statoolinfos/htmlize/PropertyStatsPage.java b/src/fr/devinsy/statoolinfos/htmlize/PropertyStatsPage.java index 22b9f29..9811ed0 100644 --- a/src/fr/devinsy/statoolinfos/htmlize/PropertyStatsPage.java +++ b/src/fr/devinsy/statoolinfos/htmlize/PropertyStatsPage.java @@ -27,8 +27,8 @@ import org.slf4j.LoggerFactory; import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosUtils; -import fr.devinsy.statoolinfos.stats.PropertyStat; -import fr.devinsy.statoolinfos.stats.PropertyStats; +import fr.devinsy.statoolinfos.stats.properties.PropertyStat; +import fr.devinsy.statoolinfos.stats.properties.PropertyStats; import fr.devinsy.statoolinfos.util.BuildInformation; import fr.devinsy.xidyn.XidynException; import fr.devinsy.xidyn.data.TagDataManager; diff --git a/src/fr/devinsy/statoolinfos/htmlize/propertiesFiles.xhtml b/src/fr/devinsy/statoolinfos/htmlize/propertiesFiles.xhtml index 6baca5c..c5666ad 100644 --- a/src/fr/devinsy/statoolinfos/htmlize/propertiesFiles.xhtml +++ b/src/fr/devinsy/statoolinfos/htmlize/propertiesFiles.xhtml @@ -15,25 +15,34 @@

Fichiers properties

Nombre de fichiers : n/a
- +
- - - - + + + + + + + + - + + + +
NomOrganisationNombre de propertySourceNomOrganisationLignesPropriétésRempliesVidesErreurs
- - + n/a + + +  n/a n/an/an/an/an/an/a
diff --git a/src/fr/devinsy/statoolinfos/properties/PathProperties.java b/src/fr/devinsy/statoolinfos/properties/PathProperties.java index 6f31784..99f979c 100644 --- a/src/fr/devinsy/statoolinfos/properties/PathProperties.java +++ b/src/fr/devinsy/statoolinfos/properties/PathProperties.java @@ -99,6 +99,16 @@ public interface PathProperties extends Iterable @Override Iterator iterator(); + /** + * Put. + * + * @param key + * the key + * @param value + * the value + */ + void put(final String key, final long value); + /** * Put. * diff --git a/src/fr/devinsy/statoolinfos/properties/PathPropertyList.java b/src/fr/devinsy/statoolinfos/properties/PathPropertyList.java index 6289551..3de37ef 100644 --- a/src/fr/devinsy/statoolinfos/properties/PathPropertyList.java +++ b/src/fr/devinsy/statoolinfos/properties/PathPropertyList.java @@ -342,6 +342,12 @@ public class PathPropertyList extends ArrayList implements PathPro return result; } + @Override + public void put(final String key, final long value) + { + put(key, "" + value); + } + /** * Put. * diff --git a/src/fr/devinsy/statoolinfos/stats/PropertiesFileStat.java b/src/fr/devinsy/statoolinfos/stats/PropertiesFileStat.java deleted file mode 100644 index 0a7d45f..0000000 --- a/src/fr/devinsy/statoolinfos/stats/PropertiesFileStat.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2020 Christian Pierre MOMON - * - * This file is part of StatoolInfos, simple service statistics tool. - * - * StatoolInfos is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * StatoolInfos is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with StatoolInfos. If not, see . - */ -package fr.devinsy.statoolinfos.stats; - -import java.net.URL; - -/** - * The Class PropertiesFileStat. - */ -public class PropertiesFileStat -{ - private URL url; - private int lineCount; - private int activeLineCount; - - /** - * Instantiates a new properties file stat. - */ - public PropertiesFileStat() - { - } -} diff --git a/src/fr/devinsy/statoolinfos/stats/PropertiesFileStats.java b/src/fr/devinsy/statoolinfos/stats/PropertiesFileStats.java deleted file mode 100644 index c6a382a..0000000 --- a/src/fr/devinsy/statoolinfos/stats/PropertiesFileStats.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2020 Christian Pierre MOMON - * - * This file is part of StatoolInfos, simple service statistics tool. - * - * StatoolInfos is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * StatoolInfos is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with StatoolInfos. If not, see . - */ -package fr.devinsy.statoolinfos.stats; - -import java.util.ArrayList; - -/** - * The Class PropertiesFileStats. - */ -public class PropertiesFileStats extends ArrayList -{ - private static final long serialVersionUID = 8828667177906801801L; - - /** - * Instantiates a new properties file stats. - */ - public PropertiesFileStats() - { - super(); - } -} diff --git a/src/fr/devinsy/statoolinfos/stats/StatAgent.java b/src/fr/devinsy/statoolinfos/stats/StatAgent.java index 0540625..aa91f27 100644 --- a/src/fr/devinsy/statoolinfos/stats/StatAgent.java +++ b/src/fr/devinsy/statoolinfos/stats/StatAgent.java @@ -18,11 +18,17 @@ */ package fr.devinsy.statoolinfos.stats; +import java.io.IOException; +import java.net.MalformedURLException; + import fr.devinsy.statoolinfos.core.Federation; import fr.devinsy.statoolinfos.core.Organization; import fr.devinsy.statoolinfos.core.Organizations; import fr.devinsy.statoolinfos.core.Service; import fr.devinsy.statoolinfos.core.Services; +import fr.devinsy.statoolinfos.crawl.CrawlCache; +import fr.devinsy.statoolinfos.stats.properties.PropertyStats; +import fr.devinsy.statoolinfos.stats.propertyfiles.PropertiesFileStats; /** * The Class StatAgent. @@ -65,6 +71,35 @@ public class StatAgent return result; } + /** + * Stat all properties files. + * + * @param federation + * the federation + * @return the properties file stats + * @throws IOException + * @throws MalformedURLException + */ + public static PropertiesFileStats statAllPropertiesFiles(final Federation federation, final CrawlCache cache) throws MalformedURLException, IOException + { + PropertiesFileStats result; + + result = new PropertiesFileStats(); + + for (Organization organization : federation.getOrganizations()) + { + result.add(result.stat(organization)); + + for (Service service : organization.getServices()) + { + result.add(result.stat(service)); + } + } + + // + return result; + } + /** * Stat federation properties. * diff --git a/src/fr/devinsy/statoolinfos/stats/PropertyStat.java b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStat.java similarity index 97% rename from src/fr/devinsy/statoolinfos/stats/PropertyStat.java rename to src/fr/devinsy/statoolinfos/stats/properties/PropertyStat.java index 0879a4a..7a072c1 100644 --- a/src/fr/devinsy/statoolinfos/stats/PropertyStat.java +++ b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStat.java @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with StatoolInfos. If not, see . */ -package fr.devinsy.statoolinfos.stats; +package fr.devinsy.statoolinfos.stats.properties; import org.apache.commons.lang3.StringUtils; diff --git a/src/fr/devinsy/statoolinfos/stats/PropertyStatComparator.java b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStatComparator.java similarity index 98% rename from src/fr/devinsy/statoolinfos/stats/PropertyStatComparator.java rename to src/fr/devinsy/statoolinfos/stats/properties/PropertyStatComparator.java index 6a02843..9d8ebe2 100644 --- a/src/fr/devinsy/statoolinfos/stats/PropertyStatComparator.java +++ b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStatComparator.java @@ -1,7 +1,7 @@ /* * */ -package fr.devinsy.statoolinfos.stats; +package fr.devinsy.statoolinfos.stats.properties; import java.util.Comparator; diff --git a/src/fr/devinsy/statoolinfos/stats/PropertyStatList.java b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStatList.java similarity index 98% rename from src/fr/devinsy/statoolinfos/stats/PropertyStatList.java rename to src/fr/devinsy/statoolinfos/stats/properties/PropertyStatList.java index 8260faa..98d3ad7 100644 --- a/src/fr/devinsy/statoolinfos/stats/PropertyStatList.java +++ b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStatList.java @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with StatoolInfos. If not, see . */ -package fr.devinsy.statoolinfos.stats; +package fr.devinsy.statoolinfos.stats.properties; import java.util.ArrayList; import java.util.Collections; diff --git a/src/fr/devinsy/statoolinfos/stats/PropertyStatSet.java b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStatSet.java similarity index 96% rename from src/fr/devinsy/statoolinfos/stats/PropertyStatSet.java rename to src/fr/devinsy/statoolinfos/stats/properties/PropertyStatSet.java index 946506e..37cd6d2 100644 --- a/src/fr/devinsy/statoolinfos/stats/PropertyStatSet.java +++ b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStatSet.java @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with StatoolInfos. If not, see . */ -package fr.devinsy.statoolinfos.stats; +package fr.devinsy.statoolinfos.stats.properties; import java.util.HashMap; diff --git a/src/fr/devinsy/statoolinfos/stats/PropertyStats.java b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStats.java similarity index 98% rename from src/fr/devinsy/statoolinfos/stats/PropertyStats.java rename to src/fr/devinsy/statoolinfos/stats/properties/PropertyStats.java index 9cda852..2a639b1 100644 --- a/src/fr/devinsy/statoolinfos/stats/PropertyStats.java +++ b/src/fr/devinsy/statoolinfos/stats/properties/PropertyStats.java @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with StatoolInfos. If not, see . */ -package fr.devinsy.statoolinfos.stats; +package fr.devinsy.statoolinfos.stats.properties; import org.apache.commons.lang3.StringUtils; diff --git a/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStat.java b/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStat.java new file mode 100644 index 0000000..726b316 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStat.java @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2020 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.propertyfiles; + +import java.net.URL; + +import fr.devinsy.statoolinfos.core.Organization; + +/** + * The Class PropertiesFileStat. + */ +public class PropertiesFileStat +{ + private String localName; + private URL url; + private Organization organization; + private int lineCount; + private int activeLineCount; + private int blankPropertyCount; + private int filledPropertyCount; + private int errorCount; + + /** + * Instantiates a new properties file stat. + */ + public PropertiesFileStat() + { + } + + public int getActiveLineCount() + { + return this.activeLineCount; + } + + public int getBlankPropertyCount() + { + return this.blankPropertyCount; + } + + public int getErrorCount() + { + return this.errorCount; + } + + public int getFilledPropertyCount() + { + return this.filledPropertyCount; + } + + public int getLineCount() + { + return this.lineCount; + } + + public String getLocalName() + { + return this.localName; + } + + public Organization getOrganization() + { + return this.organization; + } + + public URL getURL() + { + return this.url; + } + + public void incActiveLineCount() + { + this.activeLineCount += 1; + } + + public void incBlankPropertyCount() + { + this.blankPropertyCount += 1; + } + + public void incErrorCount() + { + this.errorCount += 1; + } + + public void incFilledPropertyCount() + { + this.filledPropertyCount += 1; + } + + public void incLineCount() + { + this.lineCount += 1; + } + + public void setActiveLineCount(final int activeLineCount) + { + this.activeLineCount = activeLineCount; + } + + public void setBlankPropertyCount(final int blankPropertyCount) + { + this.blankPropertyCount = blankPropertyCount; + } + + public void setErrorCount(final int errorCount) + { + this.errorCount = errorCount; + } + + public void setFilledPropertyCount(final int filledPropertyCount) + { + this.filledPropertyCount = filledPropertyCount; + } + + public void setLineCount(final int lineCount) + { + this.lineCount = lineCount; + } + + public void setLocalName(final String localName) + { + this.localName = localName; + } + + public void setOrganization(final Organization organization) + { + this.organization = organization; + } + + public void setURL(final URL url) + { + this.url = url; + } +} diff --git a/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStatComparator.java b/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStatComparator.java new file mode 100644 index 0000000..3783ce5 --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStatComparator.java @@ -0,0 +1,271 @@ +/* + * + */ +package fr.devinsy.statoolinfos.stats.propertyfiles; + +import java.util.Comparator; + +import fr.devinsy.statoolinfos.util.CompareUtils; + +/** + * The Class PropertyStatComparator. + */ +public class PropertiesFileStatComparator implements Comparator +{ + public enum Sorting + { + LOCAL_NAME, + ORGANIZATION, + LINE_COUNT, + ACTIVE_LINE_COUNT, + BLANK_PROPERTY_COUNT, + FILLED_PROPERTY_COUNT, + ERROR_COUNT + } + + private Sorting sorting; + + /** + * Instantiates a new project comparator. + * + * @param sorting + * the sorting + */ + public PropertiesFileStatComparator(final Sorting sorting) + { + // + this.sorting = sorting; + } + + /** + * Compare. + * + * @param alpha + * the alpha + * @param bravo + * the bravo + * @return the int + */ + @Override + public int compare(final PropertiesFileStat alpha, final PropertiesFileStat bravo) + { + int result; + + result = compare(alpha, bravo, this.sorting); + + // + return result; + } + + /** + * Compare. + * + * @param alpha + * the alpha + * @param bravo + * the bravo + * @param sorting + * the sorting + * @return the int + */ + public static int compare(final PropertiesFileStat alpha, final PropertiesFileStat bravo, final Sorting sorting) + { + int result; + + if (sorting == null) + { + result = 0; + } + else + { + switch (sorting) + { + default: + case LOCAL_NAME: + result = CompareUtils.compare(getLocalName(alpha), getLocalName(bravo)); + break; + + case ORGANIZATION: + result = CompareUtils.compare(getOrganizationName(alpha), getOrganizationName(bravo)); + break; + + case LINE_COUNT: + result = CompareUtils.compare(getLineCount(alpha), getLineCount(bravo)); + break; + + case ACTIVE_LINE_COUNT: + result = CompareUtils.compare(getActiveCount(alpha), getActiveCount(bravo)); + break; + + case FILLED_PROPERTY_COUNT: + result = CompareUtils.compare(getFilledCount(alpha), getFilledCount(bravo)); + break; + + case BLANK_PROPERTY_COUNT: + result = CompareUtils.compare(getBlankCount(alpha), getBlankCount(bravo)); + break; + + case ERROR_COUNT: + result = CompareUtils.compare(getErrorCount(alpha), getErrorCount(bravo)); + break; + } + } + + // + return result; + } + + /** + * Gets the active count. + * + * @param source + * the source + * @return the active count + */ + public static Long getActiveCount(final PropertiesFileStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getActiveLineCount(); + } + + // + return result; + } + + /** + * Gets the id. + * + * @param source + * the source + * @return the id + */ + public static Long getBlankCount(final PropertiesFileStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getBlankPropertyCount(); + } + + // + return result; + } + + /** + * Gets the error count. + * + * @param source + * the source + * @return the error count + */ + public static Long getErrorCount(final PropertiesFileStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getErrorCount(); + } + + // + return result; + } + + /** + * Gets the filled count. + * + * @param source + * the source + * @return the filled count + */ + public static Long getFilledCount(final PropertiesFileStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getFilledPropertyCount(); + } + + // + return result; + } + + public static Long getLineCount(final PropertiesFileStat source) + { + Long result; + + if (source == null) + { + result = null; + } + else + { + result = (long) source.getLineCount(); + } + + // + return result; + } + + public static String getLocalName(final PropertiesFileStat source) + { + String result; + + if (source == null) + { + result = null; + } + else + { + result = source.getLocalName(); + } + + // + return result; + } + + /** + * Gets the organization name. + * + * @param source + * the source + * @return the organization name + */ + public static String getOrganizationName(final PropertiesFileStat source) + { + String result; + + if (source == null) + { + result = null; + } + else + { + result = source.getOrganization().getName(); + } + + // + return result; + } +} diff --git a/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java b/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java new file mode 100644 index 0000000..01a21ce --- /dev/null +++ b/src/fr/devinsy/statoolinfos/stats/propertyfiles/PropertiesFileStats.java @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2020 Christian Pierre MOMON + * + * This file is part of StatoolInfos, simple service statistics tool. + * + * StatoolInfos is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * StatoolInfos is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with StatoolInfos. If not, see . + */ +package fr.devinsy.statoolinfos.stats.propertyfiles; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; + +import org.apache.commons.lang3.StringUtils; + +import fr.devinsy.statoolinfos.core.Organization; +import fr.devinsy.statoolinfos.core.Service; +import fr.devinsy.strings.StringList; +import fr.devinsy.strings.StringsUtils; + +/** + * The Class PropertiesFileStats. + */ +public class PropertiesFileStats extends ArrayList +{ + private static final long serialVersionUID = 8828667177906801801L; + + /** + * Instantiates a new properties file stats. + */ + public PropertiesFileStats() + { + super(); + } + + /** + * Sort. + * + * @param sorting + * the sorting + * @return the properties file stats + */ + public PropertiesFileStats sort(final PropertiesFileStatComparator.Sorting sorting) + { + PropertiesFileStats result; + + sort(new PropertiesFileStatComparator(sorting)); + + result = this; + + // + return result; + } + + /** + * Sort by blank count. + * + * @return the property stat list + */ + public PropertiesFileStats sortByName() + { + PropertiesFileStats result; + + result = sort(PropertiesFileStatComparator.Sorting.LOCAL_NAME); + + // + return result; + } + + /** + * Stat. + * + * @param url + * the url + * @param file + * the file + * @param localName + * the local name + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public PropertiesFileStat stat(final File inputFile) throws IOException + { + PropertiesFileStat result; + + result = new PropertiesFileStat(); + + StringList lines = StringsUtils.load(inputFile); + for (String line : lines) + { + result.incLineCount(); + if ((StringUtils.isNotBlank(line)) && (!line.trim().startsWith("#"))) + { + if (line.matches("^.+=.+$")) + { + result.incActiveLineCount(); + result.incFilledPropertyCount(); + } + else if (line.matches("^.+=.*$")) + { + result.incActiveLineCount(); + result.incBlankPropertyCount(); + } + else + { + result.incErrorCount(); + } + } + } + + // + return result; + } + + /** + * Stat. + * + * @param organization + * the organization + * @return the properties file stat + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public PropertiesFileStat stat(final Organization organization) throws IOException + { + PropertiesFileStat result; + + result = stat(organization.getInputFile()); + + result.setURL(organization.getInputURL()); + result.setLocalName(organization.getTechnicalName() + ".properties"); + result.setOrganization(organization); + + // + return result; + } + + /** + * Stat. + * + * @param service + * the service + * @return the properties file stat + * @throws IOException + * Signals that an I/O exception has occurred. + */ + public PropertiesFileStat stat(final Service service) throws IOException + { + PropertiesFileStat result; + + result = stat(service.getInputFile()); + + result.setURL(service.getInputURL()); + result.setLocalName(service.getOrganization().getTechnicalName() + "-" + service.getTechnicalName() + ".properties"); + result.setOrganization(service.getOrganization()); + + // + return result; + } +}