Added property files statistics.

This commit is contained in:
Christian P. MOMON 2020-09-23 21:56:40 +02:00
parent ed60bf88cd
commit da17e52d37
25 changed files with 985 additions and 174 deletions

View file

@ -189,7 +189,7 @@ public class Configuration extends PathPropertyList
* *
* @return the crawl input * @return the crawl input
*/ */
public File getCrawlInput() public File getCrawlInputFile()
{ {
File result; File result;

View file

@ -84,16 +84,15 @@ public class Factory
PathProperties properties = PathPropertyUtils.load(federationFile); PathProperties properties = PathPropertyUtils.load(federationFile);
result = new Federation(properties); result = new Federation(properties);
result.setLocalFile(federationFile); result.setInputFile(federationFile);
PathProperties subs = result.getByPrefix("subs"); PathProperties subs = result.getByPrefix("subs");
for (PathProperty property : subs) for (PathProperty property : subs)
{ {
if (StringUtils.startsWith(property.getValue(), "http")) if (StringUtils.startsWith(property.getValue(), "http"))
{ {
File subFile = cache.restoreFile(new URL(property.getValue())); URL inputURL = new URL(property.getValue());
Organization organization = loadOrganization(subFile, cache); Organization organization = loadOrganization(inputURL, cache);
organization.setFederation(result);
result.getOrganizations().add(organization); result.getOrganizations().add(organization);
} }
} }
@ -119,15 +118,53 @@ public class Factory
PathProperties properties = PathPropertyUtils.load(organizationFile); PathProperties properties = PathPropertyUtils.load(organizationFile);
result = new Organization(properties); result = new Organization(properties);
result.setLocalFile(organizationFile); result.setInputFile(organizationFile);
PathProperties subs = result.getByPrefix("subs"); PathProperties subs = result.getByPrefix("subs");
for (PathProperty property : subs) for (PathProperty property : subs)
{ {
if (StringUtils.startsWith(property.getValue(), "http")) if (StringUtils.startsWith(property.getValue(), "http"))
{ {
File subFile = cache.restoreFile(new URL(property.getValue())); URL serviceInputFile = new URL(property.getValue());
Service service = loadService(subFile); Service service = loadService(serviceInputFile, cache);
service.setOrganization(result);
result.getServices().add(service);
}
}
//
return result;
}
/**
* Load organization.
*
* @param inputURL
* the input
* @param cache
* the cache
* @return the organization
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static Organization loadOrganization(final URL inputURL, final CrawlCache cache) throws IOException
{
Organization result;
File inputFile = cache.restoreFile(inputURL);
PathProperties properties = PathPropertyUtils.load(inputFile);
result = new Organization(properties);
result.setInputFile(inputFile);
result.setInputURL(inputURL);
PathProperties subs = result.getByPrefix("subs");
for (PathProperty property : subs)
{
if (StringUtils.startsWith(property.getValue(), "http"))
{
URL serviceInputURL = new URL(property.getValue());
Service service = loadService(serviceInputURL, cache);
service.setOrganization(result); service.setOrganization(result);
result.getServices().add(service); result.getServices().add(service);
} }
@ -143,13 +180,16 @@ public class Factory
* @return the service * @return the service
* @throws IOException * @throws IOException
*/ */
public static Service loadService(final File serviceFile) throws IOException public static Service loadService(final URL inputURL, final CrawlCache cache) throws IOException
{ {
Service result; Service result;
PathProperties properties = PathPropertyUtils.load(serviceFile); File inputFile = cache.restoreFile(inputURL);
PathProperties properties = PathPropertyUtils.load(inputFile);
result = new Service(properties); result = new Service(properties);
result.setLocalFile(serviceFile); result.setInputFile(inputFile);
result.setInputURL(inputURL);
// //
return result; return result;

View file

@ -34,7 +34,7 @@ public class Federation extends PathPropertyList
{ {
private static final long serialVersionUID = -8970835291634661580L; private static final long serialVersionUID = -8970835291634661580L;
private Organizations organizations; private Organizations organizations;
private File localFile; private File inputFile;
/** /**
* Instantiates a new federation. * Instantiates a new federation.
@ -100,14 +100,9 @@ public class Federation extends PathPropertyList
return result; return result;
} }
/** public File getInputFile()
* Gets the local file.
*
* @return the local file
*/
public File getLocalFile()
{ {
return this.localFile; return this.inputFile;
} }
/** /**
@ -203,8 +198,8 @@ public class Federation extends PathPropertyList
return result; return result;
} }
public void setLocalFile(final File localFile) public void setInputFile(final File inputFile)
{ {
this.localFile = localFile; this.inputFile = inputFile;
} }
} }

View file

@ -35,7 +35,8 @@ public class Organization extends PathPropertyList
private static final long serialVersionUID = -2709210934548224213L; private static final long serialVersionUID = -2709210934548224213L;
private Federation federation; private Federation federation;
private Services services; private Services services;
private File localFile; private File inputFile;
private URL inputURL;
/** /**
* Instantiates a new organization. * Instantiates a new organization.
@ -81,9 +82,24 @@ public class Organization extends PathPropertyList
return this.federation; return this.federation;
} }
public File getLocalFile() public File getInputFile()
{ {
return this.localFile; return this.inputFile;
}
/**
* Gets the crawled input URL.
*
* @return the crawled input URL
*/
public URL getInputURL()
{
URL result;
result = this.inputURL;
//
return result;
} }
/** /**
@ -181,8 +197,13 @@ public class Organization extends PathPropertyList
this.federation = federation; this.federation = federation;
} }
public void setLocalFile(final File localFile) public void setInputFile(final File inputFile)
{ {
this.localFile = localFile; this.inputFile = inputFile;
}
public void setInputURL(final URL inputURL)
{
this.inputURL = inputURL;
} }
} }

View file

@ -34,7 +34,8 @@ public class Service extends PathPropertyList
{ {
private static final long serialVersionUID = 3629841771102288863L; private static final long serialVersionUID = 3629841771102288863L;
private Organization organization; private Organization organization;
private File localFile; private File inputFile;
private URL inputURL;
/** /**
* Instantiates a new service. * Instantiates a new service.
@ -70,9 +71,14 @@ public class Service extends PathPropertyList
return result; return result;
} }
public File getLocalFile() public File getInputFile()
{ {
return this.localFile; return this.inputFile;
}
public URL getInputURL()
{
return this.inputURL;
} }
/** /**
@ -160,9 +166,14 @@ public class Service extends PathPropertyList
return result; return result;
} }
public void setLocalFile(final File localFile) public void setInputFile(final File inputFile)
{ {
this.localFile = localFile; this.inputFile = inputFile;
}
public void setInputURL(final URL inputURL)
{
this.inputURL = inputURL;
} }
public void setOrganization(final Organization organization) public void setOrganization(final Organization organization)

View file

@ -20,9 +20,12 @@ package fr.devinsy.statoolinfos.core;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.time.Instant;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.Date; import java.util.Date;
import java.util.Iterator; import java.util.Iterator;
@ -102,6 +105,42 @@ public class StatoolInfosUtils
FileUtils.copyURLToFile(url, finalTarget); FileUtils.copyURLToFile(url, finalTarget);
} }
/**
* Epoch to local date time.
*
* @param epoch
* the epoch
* @return the local date time
*/
public static LocalDateTime epochToLocalDateTime(final long epoch)
{
LocalDateTime result;
result = Instant.ofEpochMilli(epoch).atZone(ZoneId.systemDefault()).toLocalDateTime();
//
return result;
}
/**
* File local date time.
*
* @param file
* the file
* @return the local date time
*/
public static LocalDateTime fileLocalDateTime(final File file)
{
LocalDateTime result;
long epoch = file.lastModified();
result = epochToLocalDateTime(epoch);
//
return result;
}
/** /**
* Generate cat logo. * Generate cat logo.
* *
@ -417,4 +456,27 @@ public class StatoolInfosUtils
{ {
} }
} }
/**
* Url last modified.
*
* @param url
* the url
* @return the local date time
* @throws IOException
*/
public static LocalDateTime urlLastModified(final URL url) throws IOException
{
LocalDateTime result;
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
Long epoch = connection.getLastModified();
connection.disconnect();
result = epochToLocalDateTime(epoch);
// result =
// Instant.ofEpochMilli(epoch).atZone(ZoneId.of("GMT")).toLocalDateTime();
//
return result;
}
} }

View file

@ -107,21 +107,21 @@ public class CrawlCache
/** /**
* Restore file. * Restore file.
* *
* @param url * @param key
* the url * the key
* @return the file * @return the file
*/ */
public File restoreFile(final URL url) public File restoreFile(final String key)
{ {
File result; File result;
if (url == null) if (StringUtils.isBlank(key))
{ {
throw new IllegalArgumentException("Null parameter."); throw new IllegalArgumentException("Null parameter.");
} }
else else
{ {
result = buildFile(url.toString()); result = buildFile(key);
if (!result.exists()) if (!result.exists())
{ {
result = null; result = null;
@ -132,6 +132,42 @@ public class CrawlCache
return result; return result;
} }
/**
* Restore file.
*
* @param url
* the url
* @return the file
*/
public File restoreFile(final URL url)
{
File result;
result = restoreFile(url.toString());
//
return result;
}
/**
* Restore file to.
*
* @param key
* the key
* @param target
* the target
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public void restoreFileTo(final String key, final File target) throws IOException
{
File logoFile = restoreFile(key);
if (logoFile != null)
{
FileUtils.copyFile(logoFile, target);
}
}
/** /**
* Restore file to. * Restore file to.
* *
@ -144,14 +180,7 @@ public class CrawlCache
*/ */
public void restoreFileTo(final URL url, final File target) throws IOException public void restoreFileTo(final URL url, final File target) throws IOException
{ {
if (url != null) restoreFile(url.toString());
{
File logoFile = restoreFile(url);
if (logoFile != null)
{
FileUtils.copyFile(logoFile, target);
}
}
} }
/** /**
@ -238,6 +267,33 @@ public class CrawlCache
return result; return result;
} }
/**
* Store.
*
* @param key
* the key
* @return the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public File store(final String key, final File source) throws IOException
{
File result;
if (StringUtils.isBlank(key))
{
result = null;
}
else
{
result = buildFile(key);
FileUtils.copyFile(source, result);
}
//
return result;
}
/** /**
* Store. * Store.
* *
@ -265,6 +321,36 @@ public class CrawlCache
return result; return result;
} }
/**
* Store properties.
*
* @param key
* the key
* @param properties
* the properties
* @return the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public File storeProperties(final String key, final PathProperties properties) throws IOException
{
File result;
if (StringUtils.isBlank(key))
{
result = null;
}
else
{
result = buildFile(key);
PathPropertyUtils.save(result, properties);
}
//
return result;
}
/** /**
* Store. * Store.
* *
@ -286,8 +372,7 @@ public class CrawlCache
} }
else else
{ {
String key = url.toString() + ".properties"; result = buildFile(url.toString());
result = buildFile(key);
PathPropertyUtils.save(result, properties); PathPropertyUtils.save(result, properties);
} }

View file

@ -21,8 +21,11 @@ package fr.devinsy.statoolinfos.crawl;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -30,6 +33,7 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.Configuration; import fr.devinsy.statoolinfos.core.Configuration;
import fr.devinsy.statoolinfos.core.Factory; import fr.devinsy.statoolinfos.core.Factory;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathProperties;
import fr.devinsy.statoolinfos.properties.PathProperty; import fr.devinsy.statoolinfos.properties.PathProperty;
import fr.devinsy.statoolinfos.properties.PathPropertyList; import fr.devinsy.statoolinfos.properties.PathPropertyList;
@ -89,10 +93,18 @@ public class Crawler
CrawlCache cache = configuration.getCrawlCache(); CrawlCache cache = configuration.getCrawlCache();
PathProperties input = PathPropertyUtils.load(configuration.getCrawlInput()); PathProperties input = PathPropertyUtils.load(configuration.getCrawlInputFile());
if (configuration.isFederation())
{
cache.store(input.get("federation.name"), configuration.getCrawlInputFile());
cache.storeQuietly(input.getURL("federation.logo")); cache.storeQuietly(input.getURL("federation.logo"));
}
else
{
cache.store(input.get("organization.name"), configuration.getCrawlInputFile());
cache.storeQuietly(input.getURL("organization.logo")); cache.storeQuietly(input.getURL("organization.logo"));
}
PathProperties subs = input.getByPrefix("subs"); PathProperties subs = input.getByPrefix("subs");
for (PathProperty property : subs) for (PathProperty property : subs)
@ -137,20 +149,29 @@ public class Crawler
{ {
logger.info("Crawling " + url); logger.info("Crawling " + url);
// Crawl.
File file = cache.store(url); File file = cache.store(url);
PathProperties properties = PathPropertyUtils.load(file);
// Build crawl data.
PathProperties crawlSection = new PathPropertyList(); PathProperties crawlSection = new PathPropertyList();
crawlSection.put("crawl.crawler", "StatoolInfos"); crawlSection.put("crawl.crawler", "StatoolInfos");
crawlSection.put("crawl.datetime", LocalDateTime.now().toString()); crawlSection.put("crawl.datetime", LocalDateTime.now().toString());
crawlSection.put("crawl.url", url.toString()); crawlSection.put("crawl.url", url.toString());
properties.add(crawlSection); crawlSection.put("crawl.file.size", FileUtils.sizeOf(file));
cache.storeProperties(url, properties); crawlSection.put("crawl.file.datetime", StatoolInfosUtils.urlLastModified(url).toString());
crawlSection.put("crawl.file.sha1", DigestUtils.sha1Hex(FileUtils.readFileToByteArray(file)));
// Add crawl data in crawled file.
String lines = crawlSection.toStringListFormatted().toStringSeparatedBy('\n');
FileUtils.write(file, FileUtils.readFileToString(file, StandardCharsets.UTF_8) + "\n" + lines, StandardCharsets.UTF_8);
// Crawl another resources.
PathProperties properties = PathPropertyUtils.load(file);
cache.storeQuietly(properties.getURL("organization.logo")); cache.storeQuietly(properties.getURL("organization.logo"));
cache.storeQuietly(properties.getURL("service.logo")); cache.storeQuietly(properties.getURL("service.logo"));
// // Crawl subs.
PathProperties subs = properties.getByPrefix("subs"); PathProperties subs = properties.getByPrefix("subs");
for (PathProperty property : subs) for (PathProperty property : subs)
{ {

View file

@ -35,8 +35,9 @@ import fr.devinsy.statoolinfos.core.Service;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.statoolinfos.crawl.CrawlCache; import fr.devinsy.statoolinfos.crawl.CrawlCache;
import fr.devinsy.statoolinfos.stats.PropertyStats;
import fr.devinsy.statoolinfos.stats.StatAgent; import fr.devinsy.statoolinfos.stats.StatAgent;
import fr.devinsy.statoolinfos.stats.properties.PropertyStats;
import fr.devinsy.statoolinfos.stats.propertyfiles.PropertiesFileStats;
/** /**
* The Class Htmlizer. * The Class Htmlizer.
@ -212,8 +213,8 @@ public class Htmlizer
// Manage the logo file. // Manage the logo file.
logger.info("Htmlize federation logo."); logger.info("Htmlize federation logo.");
cache.restoreLogoTo(federation.getLogoURL(), new File(htmlizeDirectory, federation.getTechnicalName() + "-logo.png"), federation.getTechnicalName()); cache.restoreLogoTo(federation.getLogoURL(), new File(htmlizeDirectory, federation.getTechnicalName() + "-logo.png"), federation.getTechnicalName());
logger.info("Htmlize federation properties file."); logger.info("Htmlize federation properties files.");
FileUtils.copyFile(federation.getLocalFile(), new File(htmlizeDirectory, federation.getTechnicalName() + ".properties")); FileUtils.copyFile(federation.getInputFile(), new File(htmlizeDirectory, federation.getTechnicalName() + ".properties"));
// //
logger.info("Htmlize about page."); logger.info("Htmlize about page.");
@ -231,7 +232,7 @@ public class Htmlizer
logger.info("Htmlize organization logo: {}.", organization.getName()); logger.info("Htmlize organization logo: {}.", organization.getName());
cache.restoreLogoTo(organization.getLogoURL(), new File(htmlizeDirectory, organization.getTechnicalName() + "-logo.png"), organization.getTechnicalName()); cache.restoreLogoTo(organization.getLogoURL(), new File(htmlizeDirectory, organization.getTechnicalName() + "-logo.png"), organization.getTechnicalName());
logger.info("Htmlize organization properties file: {}.", organization.getName()); logger.info("Htmlize organization properties file: {}.", organization.getName());
FileUtils.copyFile(organization.getLocalFile(), new File(htmlizeDirectory, organization.getTechnicalName() + ".properties")); FileUtils.copyFile(organization.getInputFile(), new File(htmlizeDirectory, organization.getTechnicalName() + ".properties"));
// //
logger.info("Htmlize organization page: {}.", organization.getName()); logger.info("Htmlize organization page: {}.", organization.getName());
@ -244,7 +245,7 @@ public class Htmlizer
logger.info("Htmlize service logo: {}.", service.getName()); logger.info("Htmlize service logo: {}.", service.getName());
cache.restoreLogoTo(service.getLogoURL(), new File(htmlizeDirectory, organization.getTechnicalName() + "-" + service.getTechnicalName() + "-logo.png"), service.getTechnicalName()); cache.restoreLogoTo(service.getLogoURL(), new File(htmlizeDirectory, organization.getTechnicalName() + "-" + service.getTechnicalName() + "-logo.png"), service.getTechnicalName());
logger.info("Htmlize service properties file: {}.", service.getName()); logger.info("Htmlize service properties file: {}.", service.getName());
FileUtils.copyFile(service.getLocalFile(), FileUtils.copyFile(service.getInputFile(),
new File(htmlizeDirectory, organization.getTechnicalName() + "-" + service.getTechnicalName() + ".properties")); new File(htmlizeDirectory, organization.getTechnicalName() + "-" + service.getTechnicalName() + ".properties"));
logger.info("Htmlize service page: {}.", service.getName()); logger.info("Htmlize service page: {}.", service.getName());
@ -258,10 +259,12 @@ public class Htmlizer
page = ServicesPage.build(federation.getAllServices()); page = ServicesPage.build(federation.getAllServices());
FileUtils.write(new File(htmlizeDirectory, "services.xhtml"), page, StandardCharsets.UTF_8); FileUtils.write(new File(htmlizeDirectory, "services.xhtml"), page, StandardCharsets.UTF_8);
{
logger.info("Htmlize propertiesFiles page."); logger.info("Htmlize propertiesFiles page.");
page = PropertiesFilesPage.build(federation); PropertiesFileStats stats = StatAgent.statAllPropertiesFiles(federation, cache).sortByName();
page = PropertiesFilesPage.build(stats);
FileUtils.write(new File(htmlizeDirectory, "propertiesFiles.xhtml"), page, StandardCharsets.UTF_8); FileUtils.write(new File(htmlizeDirectory, "propertiesFiles.xhtml"), page, StandardCharsets.UTF_8);
}
// //
{ {
logger.info("Htmlize propertyStats page."); logger.info("Htmlize propertyStats page.");

View file

@ -25,9 +25,9 @@ import java.util.Locale;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.Federation;
import fr.devinsy.statoolinfos.core.Organization;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.stats.propertyfiles.PropertiesFileStat;
import fr.devinsy.statoolinfos.stats.propertyfiles.PropertiesFileStats;
import fr.devinsy.statoolinfos.util.BuildInformation; import fr.devinsy.statoolinfos.util.BuildInformation;
import fr.devinsy.xidyn.XidynException; import fr.devinsy.xidyn.XidynException;
import fr.devinsy.xidyn.data.TagDataManager; import fr.devinsy.xidyn.data.TagDataManager;
@ -49,41 +49,37 @@ public class PropertiesFilesPage
* @throws StatoolInfosException * @throws StatoolInfosException
* the statool infos exception * the statool infos exception
*/ */
public static String build(final Federation federation) throws StatoolInfosException public static String build(final PropertiesFileStats stats) throws StatoolInfosException
{ {
String result; String result;
try try
{ {
logger.debug("Building federation page {}…", federation.getName()); logger.debug("Building propertyFiles page.");
TagDataManager data = new TagDataManager(); TagDataManager data = new TagDataManager();
data.setContent("versionsup", BuildInformation.instance().version()); data.setContent("versionsup", BuildInformation.instance().version());
data.setContent("lastUpdateDate", LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd/MM/yyyy HH':'mm", Locale.FRANCE))); data.setContent("lastUpdateDate", LocalDateTime.now().format(DateTimeFormatter.ofPattern("dd/MM/yyyy HH':'mm", Locale.FRANCE)));
data.setAttribute("federationRawButton", "href", federation.getTechnicalName() + ".properties"); data.setContent("fileCount", stats.size());
data.setAttribute("federationLogo", "src", federation.getTechnicalName() + "-logo.png");
data.setEscapedContent("federationName", federation.getName());
data.setEscapedContent("federationDescription", federation.getDescription());
data.setContent("organizationCount", federation.getOrganizations().size());
data.setContent("serviceCount", federation.getServiceCount());
// //
int index = 0; int index = 0;
for (PropertiesFileStat stat : stats)
//
//
for (Organization organization : federation.getOrganizations())
{ {
data.setAttribute("organizationListLine", index, "organizationListLineNameLink", "href", organization.getTechnicalName() + ".xhtml"); data.setAttribute("fileListLine", index, "fileListLineNameLink", "href", stat.getLocalName() + ".xhtml");
data.setAttribute("organizationListLine", index, "organizationListLineLogo", "src", organization.getTechnicalName() + "-logo.png"); data.setEscapedContent("fileListLine", index, "fileListLineNameLink", stat.getLocalName());
data.setEscapedContent("organizationListLine", index, "organizationListLineNameValue", organization.getName());
data.setEscapedContent("organizationListLine", index, "organizationListLineUrlLink", organization.getWebsite()); data.setAttribute("fileListLine", index, "fileListLineOwnerLink", "href", stat.getOrganization().getTechnicalName() + ".xhtml");
data.setAttribute("organizationListLine", index, "organizationListLineUrlLink", "href", organization.getWebsite()); data.setEscapedContent("fileListLine", index, "fileListLineNameValue", stat.getOrganization().getTechnicalName());
data.setContent("organizationListLine", index, "organizationListLineServiceCount", organization.getServiceCount()); data.setAttribute("fileListLine", index, "fileListLineOwnerLogo", "src", stat.getOrganization().getTechnicalName() + "-logo.png");
data.setContent("fileListLine", index, "fileListLineLineCount", stat.getLineCount());
data.setContent("fileListLine", index, "fileListLineActiveCount", stat.getActiveLineCount());
data.setContent("fileListLine", index, "fileListLineBlankPropertyCount", stat.getBlankPropertyCount());
data.setContent("fileListLine", index, "fileListLineFilledPropertyCount", stat.getFilledPropertyCount());
data.setContent("fileListLine", index, "fileListLineErrorCount", stat.getErrorCount());
index += 1; index += 1;
} }

View file

@ -27,8 +27,8 @@ import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.statoolinfos.stats.PropertyStat; import fr.devinsy.statoolinfos.stats.properties.PropertyStat;
import fr.devinsy.statoolinfos.stats.PropertyStats; import fr.devinsy.statoolinfos.stats.properties.PropertyStats;
import fr.devinsy.statoolinfos.util.BuildInformation; import fr.devinsy.statoolinfos.util.BuildInformation;
import fr.devinsy.xidyn.XidynException; import fr.devinsy.xidyn.XidynException;
import fr.devinsy.xidyn.data.TagDataManager; import fr.devinsy.xidyn.data.TagDataManager;

View file

@ -15,25 +15,34 @@
<h2>Fichiers properties</h2> <h2>Fichiers properties</h2>
<div>Nombre de fichiers : <span id="fileCount">n/a</span></div> <div>Nombre de fichiers : <span id="fileCount">n/a</span></div>
<div class="left"> <div class="left">
<table class="table_classic center_table sortable" style="width: 600px; margin-left: auto; margin-right: auto;"> <table class="table_classic center_table sortable" style="width: 1000px; margin-left: auto; margin-right: auto;">
<thead> <thead>
<tr> <tr>
<th style="width: 200px;">Nom</th> <th style="width: 600px;">Nom</th>
<th style="width: 200px;">Organisation</th> <th style="width: 300px;">Organisation</th>
<th style="width: 100px;">Nombre de <i>property</i></th> <th style="width: 100px;">Lignes</th>
<th style="width: 250px;">Source</th> <th style="width: 100px;">Propriétés</th>
<th style="width: 100px;">Remplies</th>
<th style="width: 100px;">Vides</th>
<th style="width: 100px;">Erreurs</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<tr id="fileListLine"> <tr id="fileListLine">
<td id="fileListLineName" style="padding-top: 0; padding-bottom: 0;"> <td id="fileListLineName" style="padding-top: 0; padding-bottom: 0;">
<a href="#" id="fileListLineNameLink"> <a href="#" id="fileListLineNameLink">n/a</a>
<img id="fileListLineLogo" src="" style="width: 26px; height: 26px; padding-top:0; padding-bottom: 0; vertical-align: middle;"/> </td>
<td id="fileListLineOwner" style="padding-top: 0; padding-bottom: 0;">
<a href="#" id="fileListLineOwnerLink">
<img id="fileListLineOwnerLogo" src="" style="width: 26px; height: 26px; padding-top:0; padding-bottom: 0; vertical-align: middle;"/>
&#160;<span id="fileListLineNameValue">n/a</span> &#160;<span id="fileListLineNameValue">n/a</span>
</a> </a>
</td> </td>
<td id="fileListLineLineCount" class="td_number">n/a</td> <td id="fileListLineLineCount" class="td_number">n/a</td>
<td id="fileListLineSourceUrl"><a href="#" id="fileListLineSourceUrlLink">n/a</a></td> <td id="fileListLineActiveCount" class="td_number">n/a</td>
<td id="fileListLineFilledPropertyCount" class="td_number">n/a</td>
<td id="fileListLineBlankPropertyCount" class="td_number">n/a</td>
<td id="fileListLineErrorCount" class="td_number">n/a</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>

View file

@ -99,6 +99,16 @@ public interface PathProperties extends Iterable<PathProperty>
@Override @Override
Iterator<PathProperty> iterator(); Iterator<PathProperty> iterator();
/**
* Put.
*
* @param key
* the key
* @param value
* the value
*/
void put(final String key, final long value);
/** /**
* Put. * Put.
* *

View file

@ -342,6 +342,12 @@ public class PathPropertyList extends ArrayList<PathProperty> implements PathPro
return result; return result;
} }
@Override
public void put(final String key, final long value)
{
put(key, "" + value);
}
/** /**
* Put. * Put.
* *

View file

@ -1,38 +0,0 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.stats;
import java.net.URL;
/**
* The Class PropertiesFileStat.
*/
public class PropertiesFileStat
{
private URL url;
private int lineCount;
private int activeLineCount;
/**
* Instantiates a new properties file stat.
*/
public PropertiesFileStat()
{
}
}

View file

@ -1,37 +0,0 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.stats;
import java.util.ArrayList;
/**
* The Class PropertiesFileStats.
*/
public class PropertiesFileStats extends ArrayList<PropertiesFileStat>
{
private static final long serialVersionUID = 8828667177906801801L;
/**
* Instantiates a new properties file stats.
*/
public PropertiesFileStats()
{
super();
}
}

View file

@ -18,11 +18,17 @@
*/ */
package fr.devinsy.statoolinfos.stats; package fr.devinsy.statoolinfos.stats;
import java.io.IOException;
import java.net.MalformedURLException;
import fr.devinsy.statoolinfos.core.Federation; import fr.devinsy.statoolinfos.core.Federation;
import fr.devinsy.statoolinfos.core.Organization; import fr.devinsy.statoolinfos.core.Organization;
import fr.devinsy.statoolinfos.core.Organizations; import fr.devinsy.statoolinfos.core.Organizations;
import fr.devinsy.statoolinfos.core.Service; import fr.devinsy.statoolinfos.core.Service;
import fr.devinsy.statoolinfos.core.Services; import fr.devinsy.statoolinfos.core.Services;
import fr.devinsy.statoolinfos.crawl.CrawlCache;
import fr.devinsy.statoolinfos.stats.properties.PropertyStats;
import fr.devinsy.statoolinfos.stats.propertyfiles.PropertiesFileStats;
/** /**
* The Class StatAgent. * The Class StatAgent.
@ -65,6 +71,35 @@ public class StatAgent
return result; return result;
} }
/**
* Stat all properties files.
*
* @param federation
* the federation
* @return the properties file stats
* @throws IOException
* @throws MalformedURLException
*/
public static PropertiesFileStats statAllPropertiesFiles(final Federation federation, final CrawlCache cache) throws MalformedURLException, IOException
{
PropertiesFileStats result;
result = new PropertiesFileStats();
for (Organization organization : federation.getOrganizations())
{
result.add(result.stat(organization));
for (Service service : organization.getServices())
{
result.add(result.stat(service));
}
}
//
return result;
}
/** /**
* Stat federation properties. * Stat federation properties.
* *

View file

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>. * along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/ */
package fr.devinsy.statoolinfos.stats; package fr.devinsy.statoolinfos.stats.properties;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;

View file

@ -1,7 +1,7 @@
/* /*
* *
*/ */
package fr.devinsy.statoolinfos.stats; package fr.devinsy.statoolinfos.stats.properties;
import java.util.Comparator; import java.util.Comparator;

View file

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>. * along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/ */
package fr.devinsy.statoolinfos.stats; package fr.devinsy.statoolinfos.stats.properties;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;

View file

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>. * along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/ */
package fr.devinsy.statoolinfos.stats; package fr.devinsy.statoolinfos.stats.properties;
import java.util.HashMap; import java.util.HashMap;

View file

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>. * along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/ */
package fr.devinsy.statoolinfos.stats; package fr.devinsy.statoolinfos.stats.properties;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;

View file

@ -0,0 +1,150 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.stats.propertyfiles;
import java.net.URL;
import fr.devinsy.statoolinfos.core.Organization;
/**
* The Class PropertiesFileStat.
*/
public class PropertiesFileStat
{
private String localName;
private URL url;
private Organization organization;
private int lineCount;
private int activeLineCount;
private int blankPropertyCount;
private int filledPropertyCount;
private int errorCount;
/**
* Instantiates a new properties file stat.
*/
public PropertiesFileStat()
{
}
public int getActiveLineCount()
{
return this.activeLineCount;
}
public int getBlankPropertyCount()
{
return this.blankPropertyCount;
}
public int getErrorCount()
{
return this.errorCount;
}
public int getFilledPropertyCount()
{
return this.filledPropertyCount;
}
public int getLineCount()
{
return this.lineCount;
}
public String getLocalName()
{
return this.localName;
}
public Organization getOrganization()
{
return this.organization;
}
public URL getURL()
{
return this.url;
}
public void incActiveLineCount()
{
this.activeLineCount += 1;
}
public void incBlankPropertyCount()
{
this.blankPropertyCount += 1;
}
public void incErrorCount()
{
this.errorCount += 1;
}
public void incFilledPropertyCount()
{
this.filledPropertyCount += 1;
}
public void incLineCount()
{
this.lineCount += 1;
}
public void setActiveLineCount(final int activeLineCount)
{
this.activeLineCount = activeLineCount;
}
public void setBlankPropertyCount(final int blankPropertyCount)
{
this.blankPropertyCount = blankPropertyCount;
}
public void setErrorCount(final int errorCount)
{
this.errorCount = errorCount;
}
public void setFilledPropertyCount(final int filledPropertyCount)
{
this.filledPropertyCount = filledPropertyCount;
}
public void setLineCount(final int lineCount)
{
this.lineCount = lineCount;
}
public void setLocalName(final String localName)
{
this.localName = localName;
}
public void setOrganization(final Organization organization)
{
this.organization = organization;
}
public void setURL(final URL url)
{
this.url = url;
}
}

View file

@ -0,0 +1,271 @@
/*
*
*/
package fr.devinsy.statoolinfos.stats.propertyfiles;
import java.util.Comparator;
import fr.devinsy.statoolinfos.util.CompareUtils;
/**
* The Class PropertyStatComparator.
*/
public class PropertiesFileStatComparator implements Comparator<PropertiesFileStat>
{
public enum Sorting
{
LOCAL_NAME,
ORGANIZATION,
LINE_COUNT,
ACTIVE_LINE_COUNT,
BLANK_PROPERTY_COUNT,
FILLED_PROPERTY_COUNT,
ERROR_COUNT
}
private Sorting sorting;
/**
* Instantiates a new project comparator.
*
* @param sorting
* the sorting
*/
public PropertiesFileStatComparator(final Sorting sorting)
{
//
this.sorting = sorting;
}
/**
* Compare.
*
* @param alpha
* the alpha
* @param bravo
* the bravo
* @return the int
*/
@Override
public int compare(final PropertiesFileStat alpha, final PropertiesFileStat bravo)
{
int result;
result = compare(alpha, bravo, this.sorting);
//
return result;
}
/**
* Compare.
*
* @param alpha
* the alpha
* @param bravo
* the bravo
* @param sorting
* the sorting
* @return the int
*/
public static int compare(final PropertiesFileStat alpha, final PropertiesFileStat bravo, final Sorting sorting)
{
int result;
if (sorting == null)
{
result = 0;
}
else
{
switch (sorting)
{
default:
case LOCAL_NAME:
result = CompareUtils.compare(getLocalName(alpha), getLocalName(bravo));
break;
case ORGANIZATION:
result = CompareUtils.compare(getOrganizationName(alpha), getOrganizationName(bravo));
break;
case LINE_COUNT:
result = CompareUtils.compare(getLineCount(alpha), getLineCount(bravo));
break;
case ACTIVE_LINE_COUNT:
result = CompareUtils.compare(getActiveCount(alpha), getActiveCount(bravo));
break;
case FILLED_PROPERTY_COUNT:
result = CompareUtils.compare(getFilledCount(alpha), getFilledCount(bravo));
break;
case BLANK_PROPERTY_COUNT:
result = CompareUtils.compare(getBlankCount(alpha), getBlankCount(bravo));
break;
case ERROR_COUNT:
result = CompareUtils.compare(getErrorCount(alpha), getErrorCount(bravo));
break;
}
}
//
return result;
}
/**
* Gets the active count.
*
* @param source
* the source
* @return the active count
*/
public static Long getActiveCount(final PropertiesFileStat source)
{
Long result;
if (source == null)
{
result = null;
}
else
{
result = (long) source.getActiveLineCount();
}
//
return result;
}
/**
* Gets the id.
*
* @param source
* the source
* @return the id
*/
public static Long getBlankCount(final PropertiesFileStat source)
{
Long result;
if (source == null)
{
result = null;
}
else
{
result = (long) source.getBlankPropertyCount();
}
//
return result;
}
/**
* Gets the error count.
*
* @param source
* the source
* @return the error count
*/
public static Long getErrorCount(final PropertiesFileStat source)
{
Long result;
if (source == null)
{
result = null;
}
else
{
result = (long) source.getErrorCount();
}
//
return result;
}
/**
* Gets the filled count.
*
* @param source
* the source
* @return the filled count
*/
public static Long getFilledCount(final PropertiesFileStat source)
{
Long result;
if (source == null)
{
result = null;
}
else
{
result = (long) source.getFilledPropertyCount();
}
//
return result;
}
public static Long getLineCount(final PropertiesFileStat source)
{
Long result;
if (source == null)
{
result = null;
}
else
{
result = (long) source.getLineCount();
}
//
return result;
}
public static String getLocalName(final PropertiesFileStat source)
{
String result;
if (source == null)
{
result = null;
}
else
{
result = source.getLocalName();
}
//
return result;
}
/**
* Gets the organization name.
*
* @param source
* the source
* @return the organization name
*/
public static String getOrganizationName(final PropertiesFileStat source)
{
String result;
if (source == null)
{
result = null;
}
else
{
result = source.getOrganization().getName();
}
//
return result;
}
}

View file

@ -0,0 +1,171 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.stats.propertyfiles;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import org.apache.commons.lang3.StringUtils;
import fr.devinsy.statoolinfos.core.Organization;
import fr.devinsy.statoolinfos.core.Service;
import fr.devinsy.strings.StringList;
import fr.devinsy.strings.StringsUtils;
/**
* The Class PropertiesFileStats.
*/
public class PropertiesFileStats extends ArrayList<PropertiesFileStat>
{
private static final long serialVersionUID = 8828667177906801801L;
/**
* Instantiates a new properties file stats.
*/
public PropertiesFileStats()
{
super();
}
/**
* Sort.
*
* @param sorting
* the sorting
* @return the properties file stats
*/
public PropertiesFileStats sort(final PropertiesFileStatComparator.Sorting sorting)
{
PropertiesFileStats result;
sort(new PropertiesFileStatComparator(sorting));
result = this;
//
return result;
}
/**
* Sort by blank count.
*
* @return the property stat list
*/
public PropertiesFileStats sortByName()
{
PropertiesFileStats result;
result = sort(PropertiesFileStatComparator.Sorting.LOCAL_NAME);
//
return result;
}
/**
* Stat.
*
* @param url
* the url
* @param file
* the file
* @param localName
* the local name
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public PropertiesFileStat stat(final File inputFile) throws IOException
{
PropertiesFileStat result;
result = new PropertiesFileStat();
StringList lines = StringsUtils.load(inputFile);
for (String line : lines)
{
result.incLineCount();
if ((StringUtils.isNotBlank(line)) && (!line.trim().startsWith("#")))
{
if (line.matches("^.+=.+$"))
{
result.incActiveLineCount();
result.incFilledPropertyCount();
}
else if (line.matches("^.+=.*$"))
{
result.incActiveLineCount();
result.incBlankPropertyCount();
}
else
{
result.incErrorCount();
}
}
}
//
return result;
}
/**
* Stat.
*
* @param organization
* the organization
* @return the properties file stat
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public PropertiesFileStat stat(final Organization organization) throws IOException
{
PropertiesFileStat result;
result = stat(organization.getInputFile());
result.setURL(organization.getInputURL());
result.setLocalName(organization.getTechnicalName() + ".properties");
result.setOrganization(organization);
//
return result;
}
/**
* Stat.
*
* @param service
* the service
* @return the properties file stat
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public PropertiesFileStat stat(final Service service) throws IOException
{
PropertiesFileStat result;
result = stat(service.getInputFile());
result.setURL(service.getInputURL());
result.setLocalName(service.getOrganization().getTechnicalName() + "-" + service.getTechnicalName() + ".properties");
result.setOrganization(service.getOrganization());
//
return result;
}
}