statool/statoolinfosweb
statool/statoolinfosweb
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Activity

Refactored crawl code. Add crawl journal page.

Browse source
This commit is contained in:
Christian P. MOMON 2021-05-17 13:24:10 +02:00
parent 47d4a62ac7
commit bf81404746
24 changed files with 1600 additions and 398 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
src/fr/devinsy/statoolinfos/HtmlizerContext.java
View file

@ -27,6 +27,9 @@ import fr.devinsy.statoolinfos.core.Factory;
import fr.devinsy.statoolinfos.core.Federation; import fr.devinsy.statoolinfos.core.Federation;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.crawl.CrawlCache; import fr.devinsy.statoolinfos.crawl.CrawlCache;
import fr.devinsy.statoolinfos.crawl.CrawlJournal;
import fr.devinsy.statoolinfos.crawl.CrawlJournalFile;
import fr.devinsy.statoolinfos.crawl.Crawler;
/** /**
* The Class Manager. * The Class Manager.
@ -44,6 +47,7 @@ public class HtmlizerContext
private Federation federation; private Federation federation;
private Categories categories; private Categories categories;
private CrawlCache cache; private CrawlCache cache;
private CrawlJournal crawlJournal;
/** /**
* Instantiates a new manager. * Instantiates a new manager.
@ -65,20 +69,23 @@ public class HtmlizerContext
this.configuration = Factory.loadConfiguration(configurationFile); this.configuration = Factory.loadConfiguration(configurationFile);
logger.info("Cache setting: {}", this.configuration.getCrawlCachePath()); logger.info("Cache setting: {}", this.configuration.getCrawlCachePath());
logger.info("Htmlize input setting: {}", this.configuration.getHtmlizeInputPath()); logger.info("Htmlize input setting: {}", this.configuration.getHtmlizeInputURL());
logger.info("Htmlize directory setting: {}", this.configuration.getHtmlizeDirectoryPath()); logger.info("Htmlize directory setting: {}", this.configuration.getHtmlizeDirectoryPath());
File htmlizeInput = this.configuration.getHtmlizeInput(); this.cache = new CrawlCache(this.configuration.getCrawlCacheDirectory());
this.crawlJournal = CrawlJournalFile.load(this.cache.restoreFile(Crawler.getJournalURL()));
File htmlizeInputFile = this.cache.restoreFile(this.configuration.getHtmlizeInputURL());
File htmlizeDirectory = this.configuration.getHtmlizeDirectory(); File htmlizeDirectory = this.configuration.getHtmlizeDirectory();
if (htmlizeInput == null) if (htmlizeInputFile == null)
{ {
throw new IllegalArgumentException("Htmlize input undefined."); throw new IllegalArgumentException("Htmlize input undefined.");
} }
else if (!htmlizeInput.exists()) else if (!htmlizeInputFile.exists())
{ {
throw new IllegalArgumentException("Htmlize input is missing."); throw new IllegalArgumentException("Htmlize input is missing.");
} }
else if (htmlizeInput.isDirectory()) else if (htmlizeInputFile.isDirectory())
{ {
throw new IllegalArgumentException("Htmlize input is a directory."); throw new IllegalArgumentException("Htmlize input is a directory.");
} }
@ -98,8 +105,7 @@ public class HtmlizerContext
{ {
if (this.configuration.isFederation()) if (this.configuration.isFederation())
{ {
this.cache = this.configuration.getCrawlCache(); this.federation = Factory.loadFederation(htmlizeInputFile, this.cache);
this.federation = Factory.loadFederation(this.configuration.getHtmlizeInput(), this.cache);
this.categories = Factory.loadCategories(this.configuration.getCategoryFile(), this.federation); this.categories = Factory.loadCategories(this.configuration.getCategoryFile(), this.federation);
} }
else else
@ -154,6 +160,11 @@ public class HtmlizerContext
return result; return result;
} }
public CrawlJournal getCrawlJournal()
{
return this.crawlJournal;
}
/** /**
* Gets the federation. * Gets the federation.
* *

2
src/fr/devinsy/statoolinfos/checker/PropertyChecker.java
View file

@ -95,7 +95,7 @@ public class PropertyChecker
this.federationRules.add(METRICS_WEEKS, WEEKS, PropertyMode.OPTIONAL); this.federationRules.add(METRICS_WEEKS, WEEKS, PropertyMode.OPTIONAL);
this.federationRules.add(METRICS_DAYS, DAYS, PropertyMode.OPTIONAL); this.federationRules.add(METRICS_DAYS, DAYS, PropertyMode.OPTIONAL);
// this.federationRules.add(CRAWL, ALL, PropertyMode.MANDATORY); this.federationRules.add(CRAWL, ALL, PropertyMode.MANDATORY);
// //
this.organizationRules = new PropertyRules(); this.organizationRules = new PropertyRules();

100
src/fr/devinsy/statoolinfos/core/Configuration.java
View file

@ -19,10 +19,11 @@
package fr.devinsy.statoolinfos.core; package fr.devinsy.statoolinfos.core;
import java.io.File; import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import fr.devinsy.statoolinfos.crawl.CrawlCache;
import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathProperties;
import fr.devinsy.statoolinfos.properties.PathPropertyList; import fr.devinsy.statoolinfos.properties.PathPropertyList;
import fr.devinsy.strings.StringList; import fr.devinsy.strings.StringList;
@ -177,18 +178,15 @@ public class Configuration extends PathPropertyList
} }
/** /**
* Gets the cache. * Gets the crawl cache directory.
* *
* @return the cache * @return the crawl cache directory
* @throws StatoolInfosException
*/ */
public CrawlCache getCrawlCache() throws StatoolInfosException public File getCrawlCacheDirectory()
{ {
CrawlCache result; File result;
String path = getCrawlCachePath(); result = new File(get("conf.crawl.cache"));
result = new CrawlCache(new File(path));
// //
return result; return result;
@ -209,29 +207,6 @@ public class Configuration extends PathPropertyList
return result; return result;
} }
/**
* Gets the crawl input.
*
* @return the crawl input
*/
public File getCrawlInputFile()
{
File result;
String path = getCrawlInputPath();
if (StringUtils.isBlank(path))
{
result = null;
}
else
{
result = new File(path);
}
//
return result;
}
/** /**
* Gets the crawl input path. * Gets the crawl input path.
* *
@ -247,6 +222,37 @@ public class Configuration extends PathPropertyList
return result; return result;
} }
/**
* Gets the crawl input.
*
* @return the crawl input
*/
public URL getCrawlInputURL()
{
URL result;
try
{
String path = getCrawlInputPath();
if (StringUtils.isBlank(path))
{
result = null;
}
else
{
result = new URL(path);
}
}
catch (MalformedURLException exception)
{
exception.printStackTrace();
result = null;
}
//
return result;
}
/** /**
* Gets the edito directory. * Gets the edito directory.
* *
@ -309,39 +315,17 @@ public class Configuration extends PathPropertyList
return result; return result;
} }
/**
* Gets the htmlize input.
*
* @return the htmlize input
*/
public File getHtmlizeInput()
{
File result;
String path = getHtmlizeInputPath();
if (StringUtils.isBlank(path))
{
result = null;
}
else
{
result = new File(path);
}
//
return result;
}
/** /**
* Gets the htmlize input path. * Gets the htmlize input path.
* *
* @return the htmlize input path * @return the htmlize input path
* @throws MalformedURLException
*/ */
public String getHtmlizeInputPath() public URL getHtmlizeInputURL() throws MalformedURLException
{ {
String result; URL result;
result = get("conf.htmlize.input"); result = new URL(get("conf.htmlize.input"));
// //
return result; return result;

125
src/fr/devinsy/statoolinfos/core/PropertyClassType.java Normal file
View file

@ -0,0 +1,125 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.core;
import org.apache.commons.lang3.StringUtils;
public enum PropertyClassType
{
FEDERATION,
ORGANIZATION,
SERVICE,
METRICS;
/**
* Checks if is child of.
*
* @param parent
* the parent
* @return true, if is child of
*/
public boolean isChildOf(final PropertyClassType parent)
{
boolean result;
switch (this)
{
case FEDERATION:
if (parent == null)
{
result = true;
}
else
{
result = false;
}
break;
case ORGANIZATION:
if (parent == FEDERATION)
{
result = true;
}
else
{
result = false;
}
break;
case SERVICE:
if (parent == ORGANIZATION)
{
result = true;
}
else
{
result = false;
}
break;
case METRICS:
result = true;
break;
default:
result = false;
}
//
return result;
}
/**
* Of.
*
* @param value
* the value
* @return the property class type
*/
public static PropertyClassType of(final String value)
{
PropertyClassType result;
String target = StringUtils.trim(StringUtils.toRootLowerCase(value));
if (target == null)
{
result = null;
}
else if (StringUtils.equals(target, "federation"))
{
result = FEDERATION;
}
else if (StringUtils.equals(target, "organization"))
{
result = ORGANIZATION;
}
else if (StringUtils.equals(target, "service"))
{
result = SERVICE;
}
else if (StringUtils.equals(target, "metrics"))
{
result = METRICS;
}
else
{
result = null;
}
//
return result;
}
}

7
src/fr/devinsy/statoolinfos/core/StatoolInfos.java
View file

@ -74,7 +74,7 @@ public class StatoolInfos
Configuration configuration = Factory.loadConfiguration(configurationFile); Configuration configuration = Factory.loadConfiguration(configurationFile);
Builder.clear(configuration); Builder.clear(configuration);
Crawler.clear(configuration); new Crawler(configuration.getCrawlCacheDirectory()).clear();
Htmlizer.clear(configuration); Htmlizer.clear(configuration);
} }
@ -90,7 +90,10 @@ public class StatoolInfos
*/ */
public static void crawl(final File configurationFile) throws StatoolInfosException, IOException public static void crawl(final File configurationFile) throws StatoolInfosException, IOException
{ {
Crawler.crawl(configurationFile); Configuration configuration = Factory.loadConfiguration(configurationFile);
Crawler crawler = new Crawler(configuration.getCrawlCacheDirectory());
crawler.crawl(configuration.getCrawlInputURL());
crawler.storeJournal();
} }
/** /**

23
src/fr/devinsy/statoolinfos/core/StatoolInfosUtils.java
View file

@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@ -40,6 +41,7 @@ import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager; import javax.net.ssl.X509TrustManager;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.math.NumberUtils;
@ -119,7 +121,7 @@ public class StatoolInfosUtils
// Because Tika failed to recognize SVG file without xml header // Because Tika failed to recognize SVG file without xml header
// line. // line.
if (result.equals(".txt") && (StringUtils.startsWithIgnoreCase(FileUtils.readFileToString(file, "UTF8"), "<svg"))) if (result.equals(".txt") && (StringUtils.startsWithIgnoreCase(FileUtils.readFileToString(file, StandardCharsets.UTF_8), "<svg")))
{ {
result = ".svg"; result = ".svg";
} }
@ -221,6 +223,25 @@ public class StatoolInfosUtils
return result; return result;
} }
/**
* Sha1sum a file.
*
* @param file
* the file
* @return the string
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static String sha1sum(final File file) throws IOException
{
String result;
result = DigestUtils.sha1Hex(FileUtils.readFileToByteArray(file));
//
return result;
}
/** /**
* Split day values. * Split day values.
* *

223
src/fr/devinsy/statoolinfos/crawl/CrawlCache.java
View file

@ -21,7 +21,6 @@ package fr.devinsy.statoolinfos.crawl;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
import java.nio.file.Files;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
@ -69,11 +68,11 @@ public class CrawlCache
} }
else if (StringUtils.isBlank(directory.getName())) else if (StringUtils.isBlank(directory.getName()))
{ {
throw new IllegalArgumentException("Undefined directory."); throw new IllegalArgumentException("Crawl cache directory undefined.");
} }
else if (!directory.exists()) else if (!directory.exists())
{ {
throw new IllegalArgumentException("Directory does not exist."); throw new IllegalArgumentException("Crawl cache directory does not exist.");
} }
else else
{ {
@ -88,11 +87,11 @@ public class CrawlCache
* the key * the key
* @return the file * @return the file
*/ */
private File buildFile(final String key) private File buildFile(final URL url)
{ {
File result; File result;
result = new File(this.directory, DigestUtils.md5Hex(key)); result = new File(this.directory, DigestUtils.md5Hex(url.toString()));
// //
return result; return result;
@ -113,6 +112,11 @@ public class CrawlCache
} }
} }
public File getDirectory()
{
return this.directory;
}
/** /**
* Gets the extension. * Gets the extension.
* *
@ -138,39 +142,11 @@ public class CrawlCache
return result; return result;
} }
/**
* Restore file.
*
* @param key
* the key
* @return the file
*/
public File restoreFile(final String key)
{
File result;
if (StringUtils.isBlank(key))
{
result = null;
}
else
{
result = buildFile(key);
if (!result.exists())
{
result = null;
}
}
//
return result;
}
/** /**
* Restore file. * Restore file.
* *
* @param url * @param url
* the url * the key
* @return the file * @return the file
*/ */
public File restoreFile(final URL url) public File restoreFile(final URL url)
@ -183,32 +159,17 @@ public class CrawlCache
} }
else else
{ {
result = restoreFile(url.toString()); result = buildFile(url);
if (!result.exists())
{
result = null;
}
} }
// //
return result; return result;
} }
/**
* Restore file to.
*
* @param key
* the key
* @param target
* the target
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public void restoreFileTo(final String key, final File target) throws IOException
{
File logoFile = restoreFile(key);
if (logoFile != null)
{
FileUtils.copyFile(logoFile, target);
}
}
/** /**
* Restore file to. * Restore file to.
* *
@ -221,7 +182,11 @@ public class CrawlCache
*/ */
public void restoreFileTo(final URL url, final File target) throws IOException public void restoreFileTo(final URL url, final File target) throws IOException
{ {
restoreFile(url.toString()); File source = restoreFile(url);
if (source != null)
{
FileUtils.copyFile(source, target);
}
} }
/** /**
@ -246,34 +211,13 @@ public class CrawlCache
} }
else else
{ {
if (url == null) logger.info("CatGeneratoring from {}", url);
{
try
{
logger.info("CatGeneratoring cat avatar (1): {}", target.getAbsoluteFile());
if ((generator == null) || (generator == DefaultLogoGenerator.CAT))
{
CatGenerator.buildAvatarTo(seed, target);
}
else
{
BirdGenerator.buildAvatarTo(seed, target);
}
}
catch (IOException exception)
{
logger.warn("CatGeneratoring failed for {}: {}", seed, exception.getMessage());
URLUtils.copyResource("/fr/devinsy/statoolinfos/htmlize/stuff/default-organization-logo.png", target);
}
}
else
{
File logoFile = restoreFile(url); File logoFile = restoreFile(url);
if (logoFile == null) if (logoFile == null)
{ {
try try
{ {
logger.info("CatGeneratoring cat avatar (2): {}", target.getAbsoluteFile()); logger.info("CatGeneratoring cat avatar: {}", target.getAbsoluteFile());
if ((generator == null) || (generator == DefaultLogoGenerator.CAT)) if ((generator == null) || (generator == DefaultLogoGenerator.CAT))
{ {
CatGenerator.buildAvatarTo(seed, target); CatGenerator.buildAvatarTo(seed, target);
@ -295,34 +239,6 @@ public class CrawlCache
} }
} }
} }
}
/**
* Restore properties.
*
* @param url
* the url
* @return the path property list
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public PathProperties restoreProperties(final URL url) throws IOException
{
PathProperties result;
if (url == null)
{
result = new PathPropertyList();
}
else
{
File file = buildFile(url.toString() + ".properties");
result = PathPropertyUtils.load(file);
}
//
return result;
}
/** /**
* @param url * @param url
@ -339,7 +255,7 @@ public class CrawlCache
} }
else else
{ {
File file = buildFile(url.toString()); File file = buildFile(url);
result = PathPropertyUtils.load(file); result = PathPropertyUtils.load(file);
} }
@ -347,104 +263,29 @@ public class CrawlCache
return result; return result;
} }
/**
* Store.
*
* @param key
* the key
* @return the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public File store(final String key, final File source) throws IOException
{
File result;
if (StringUtils.isBlank(key))
{
result = null;
}
else
{
result = buildFile(key);
FileUtils.copyFile(source, result);
}
//
return result;
}
/** /**
* Store. * Store.
* *
* @param url * @param url
* the url * the url
* @return the file * @param source
* @throws IOException * the source
*/
public File store(final URL url) throws IOException
{
File result;
if (StringUtils.startsWith(url.getProtocol(), "http"))
{
final int TIMEOUT = 5000;
result = buildFile(url.toString());
File temp = Files.createTempFile("tmp-", ".statoolsinfos").toFile();
FileUtils.copyURLToFile(url, temp, TIMEOUT, TIMEOUT);
if (temp.length() == 0)
{
if (result.exists())
{
logger.warn("WARNING: empty file crawled and ignored for [{}]", url);
result = null;
}
else
{
logger.warn("WARNING: empty file crawled and copied for [{}]", url);
FileUtils.copyFile(temp, result);
}
}
else
{
FileUtils.copyFile(temp, result);
}
temp.delete();
}
else
{
logger.warn("WARNING: crawl failed because bad http+ protocol for [{}]", url);
result = null;
}
//
return result;
}
/**
* Store properties.
*
* @param key
* the key
* @param properties
* the properties
* @return the file * @return the file
* @throws IOException * @throws IOException
* Signals that an I/O exception has occurred. * Signals that an I/O exception has occurred.
*/ */
public File storeProperties(final String key, final PathProperties properties) throws IOException public File store(final URL url, final File source) throws IOException
{ {
File result; File result;
if (StringUtils.isBlank(key)) if ((url == null) || (!StringUtils.startsWith(url.getProtocol(), "http")))
{ {
result = null; result = null;
} }
else else
{ {
result = buildFile(key); result = buildFile(url);
FileUtils.copyFile(source, result);
PathPropertyUtils.save(result, properties);
} }
// //
@ -472,7 +313,7 @@ public class CrawlCache
} }
else else
{ {
result = buildFile(url.toString()); result = buildFile(url);
PathPropertyUtils.save(result, properties); PathPropertyUtils.save(result, properties);
} }
@ -488,7 +329,7 @@ public class CrawlCache
* the url * the url
* @return the file * @return the file
*/ */
public File storeQuietly(final URL url) public File storeQuietly2(final URL url)
{ {
File result; File result;
@ -501,7 +342,7 @@ public class CrawlCache
else else
{ {
final int TIMEOUT = 5000; final int TIMEOUT = 5000;
result = buildFile(url.toString()); result = buildFile(url);
FileUtils.copyURLToFile(url, result, TIMEOUT, TIMEOUT); FileUtils.copyURLToFile(url, result, TIMEOUT, TIMEOUT);
logger.info("Crawled {}", url); logger.info("Crawled {}", url);
} }

62
src/fr/devinsy/statoolinfos/crawl/CrawlJournal.java Normal file
View file

@ -0,0 +1,62 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.crawl;
import java.time.LocalDateTime;
/**
* The Class CrawlJournal.
*/
public class CrawlJournal extends CrawlLogs
{
private static final long serialVersionUID = -7855320365496351766L;
private LocalDateTime datetime;
/**
* Instantiates a new crawl journal.
*/
public CrawlJournal()
{
super();
this.datetime = LocalDateTime.now();
}
/**
* Gets the date.
*
* @return the date
*/
public LocalDateTime getDatetime()
{
return this.datetime;
}
/**
* Sets the date.
*
* @param date
* the new date
*/
public void setDatetime(final LocalDateTime date)
{
this.datetime = date;
}
}

236
src/fr/devinsy/statoolinfos/crawl/CrawlJournalFile.java Normal file
View file

@ -0,0 +1,236 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.crawl;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class CrawlFile.
*/
public class CrawlJournalFile
{
private static Logger logger = LoggerFactory.getLogger(CrawlJournalFile.class);
/**
* Instantiates a new crawl file.
*/
private CrawlJournalFile()
{
super();
}
/**
* Load.
*
* @param file
* the file
* @return the path property list
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static CrawlJournal load(final File file) throws IOException
{
CrawlJournal result;
result = load(file, StandardCharsets.UTF_8);
//
return result;
}
/**
* Load.
*
* @param file
* the file
* @param charset
* the charset name
* @return the path properties
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static CrawlJournal load(final File file, final Charset charset) throws IOException
{
CrawlJournal result;
if (file == null)
{
throw new IllegalArgumentException("File parameter is null.");
}
else
{
BufferedReader in = null;
try
{
in = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
result = read(in);
}
finally
{
IOUtils.closeQuietly(in);
}
result.setDatetime(LocalDateTime.ofEpochSecond(file.lastModified() / 1000, 0, ZoneOffset.UTC));
}
//
return result;
}
/**
* Read.
*
* @param in
* the in
* @return the crawl logs
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static CrawlJournal read(final BufferedReader in) throws IOException
{
CrawlJournal result;
result = new CrawlJournal();
boolean ended = false;
while (!ended)
{
String line = in.readLine();
if (line == null)
{
ended = true;
}
else
{
CrawlLog log = valueOf(line);
result.add(log);
}
}
//
return result;
}
/**
* Save.
*
* @param file
* the file
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void save(final File file, final CrawlJournal source) throws IOException
{
PrintWriter out = null;
try
{
out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8"));
write(out, source);
}
finally
{
//
IOUtils.closeQuietly(out);
}
}
/**
* Value of.
*
* @param line
* the line
* @return the path property
*/
public static CrawlLog valueOf(final String line)
{
CrawlLog result;
if (line == null)
{
result = null;
}
else
{
String[] tokens = line.split(" ", 2);
CrawlStatus status = CrawlStatus.valueOf(tokens[0].toUpperCase());
URL url;
try
{
url = new URL(tokens[1].trim());
}
catch (MalformedURLException exception)
{
logger.error("Error valuing [{}]", line);
exception.printStackTrace();
url = null;
}
result = new CrawlLog(url, status);
}
//
return result;
}
/**
* Write.
*
* @param out
* the out
* @param source
* the source
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void write(final PrintWriter out, final CrawlJournal journal) throws IOException
{
if (journal != null)
{
for (CrawlLog log : journal)
{
String line = log.getStatus() + " " + log.getUrl();
out.write(line);
out.write("\n");
}
}
}
}

54
src/fr/devinsy/statoolinfos/crawl/CrawlLog.java Normal file
View file

@ -0,0 +1,54 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.crawl;
import java.net.URL;
/**
* The Class CrawlLog.
*/
public class CrawlLog
{
private URL url;
private CrawlStatus status;
/**
* Instantiates a new crawl log.
*
* @param url
* the url
* @param status
* the status
*/
public CrawlLog(final URL url, final CrawlStatus status)
{
this.url = url;
this.status = status;
}
public CrawlStatus getStatus()
{
return this.status;
}
public URL getUrl()
{
return this.url;
}
}

142
src/fr/devinsy/statoolinfos/crawl/CrawlLogs.java Normal file
View file

@ -0,0 +1,142 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.crawl;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import org.apache.commons.lang3.StringUtils;
/**
* The Class CrawlLogs.
*/
public class CrawlLogs extends ArrayList<CrawlLog>
{
private static final long serialVersionUID = -8749217049690008582L;
/**
* Instantiates a new crawl logs.
*/
public CrawlLogs()
{
super();
}
/**
* Adds the.
*
* @param url
* the url
* @param status
* the status
*/
public void add(final URL url, final CrawlStatus status)
{
this.add(new CrawlLog(url, status));
}
/**
* Find by software.
*
* @param softwareName
* the software name
* @return the category
*/
public CrawlLogs findByUrl(final URL url)
{
CrawlLogs result;
result = new CrawlLogs();
for (CrawlLog log : this)
{
if (StringUtils.equals(log.getUrl().toString(), url.toString()))
{
result.add(log);
}
}
//
return result;
}
/**
* Gets the errors.
*
* @return the errors
*/
public CrawlLogs getErrors()
{
CrawlLogs result;
result = new CrawlLogs();
for (CrawlLog log : this)
{
if (log.getStatus().isError())
{
result.add(log);
}
}
//
return result;
}
/**
* Gets the success.
*
* @return the success
*/
public CrawlLogs getSuccess()
{
CrawlLogs result;
result = new CrawlLogs();
for (CrawlLog log : this)
{
if (!log.getStatus().isError())
{
result.add(log);
}
}
//
return result;
}
/**
* Reverse.
*
* @return the categories
*/
public CrawlLogs reverse()
{
CrawlLogs result;
Collections.reverse(this);
result = this;
//
return result;
}
}

50
src/fr/devinsy/statoolinfos/crawl/CrawlStatus.java Normal file
View file

@ -0,0 +1,50 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.crawl;
public enum CrawlStatus
{
BADCHILDCLASS,
BADURLFORMAT,
CONNECTERROR,
DOWNLOADERROR,
EMPTY,
IOERROR,
MISSING,
SUCCESS,
UPDATED,
URLNOTFOUND;
public boolean isError()
{
boolean result;
if ((this == CrawlStatus.SUCCESS) || (this == CrawlStatus.UPDATED))
{
result = false;
}
else
{
result = true;
}
//
return result;
}
}

395
src/fr/devinsy/statoolinfos/crawl/Crawler.java
View file

@ -21,18 +21,19 @@ package fr.devinsy.statoolinfos.crawl;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.core.Configuration; import fr.devinsy.statoolinfos.core.PropertyClassType;
import fr.devinsy.statoolinfos.core.Factory;
import fr.devinsy.statoolinfos.core.StatoolInfosException; import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.core.StatoolInfosUtils; import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
import fr.devinsy.statoolinfos.properties.PathProperties; import fr.devinsy.statoolinfos.properties.PathProperties;
@ -47,107 +48,48 @@ public class Crawler
{ {
private static Logger logger = LoggerFactory.getLogger(Crawler.class); private static Logger logger = LoggerFactory.getLogger(Crawler.class);
private CrawlCache cache;
private CrawlJournal journal;
/**
* Instantiates a new crawler.
*
* @param rootDirectory
* the root directory
* @throws StatoolInfosException
* the statool infos exception
*/
public Crawler(final File rootDirectory) throws StatoolInfosException
{
logger.info("Crawl cache setting: {}", rootDirectory);
this.cache = new CrawlCache(rootDirectory);
this.journal = new CrawlJournal();
}
/** /**
* Clear. * Clear.
* *
* @param configuration
* the configuration
* @throws StatoolInfosException * @throws StatoolInfosException
* the statool infos exception * the statool infos exception
* @throws IOException
* Signals that an I/O exception has occurred.
*/ */
public static void clear(final Configuration configuration) throws StatoolInfosException, IOException public void clear() throws StatoolInfosException
{ {
logger.info("Cache setting: {}", configuration.getCrawlCachePath()); this.cache.clear();
String path = configuration.getCrawlCachePath();
if (StringUtils.isBlank(path))
{
logger.warn("Undefined crawl cache.");
}
else if (!new File(path).exists())
{
logger.warn("Crawl cache does not exist: {}.", path);
}
else
{
CrawlCache cache = configuration.getCrawlCache();
cache.clear();
}
} }
/** /**
* Crawl. * Crawl.
* *
* @param configuration * @param url
* the configuration * the input url
* @throws StatoolInfosException * @throws StatoolInfosException
* the statool infos exception * the statool infos exception
* @throws IOException * @throws IOException
* Signals that an I/O exception has occurred. * Signals that an I/O exception has occurred.
*/ */
public static void crawl(final Configuration configuration) throws StatoolInfosException, IOException public void crawl(final URL url) throws StatoolInfosException, IOException
{ {
logger.info("Crawl input setting: {}", configuration.getCrawlInputPath()); crawl(url, null);
logger.info("Crawl cache setting: {}", configuration.getCrawlCachePath());
CrawlCache cache = configuration.getCrawlCache();
PathProperties input = PathPropertyUtils.load(configuration.getCrawlInputFile());
if (configuration.isFederation())
{
cache.store(input.get("federation.name"), configuration.getCrawlInputFile());
cache.storeQuietly(input.getURL("federation.logo"));
}
else if (configuration.isOrganization())
{
cache.store(input.get("organization.name"), configuration.getCrawlInputFile());
cache.storeQuietly(input.getURL("organization.logo"));
}
PathProperties subs = input.getByPrefix("subs");
for (PathProperty property : subs)
{
if (StringUtils.isNotBlank(property.getValue()))
{
try
{
URL subUrl = new URL(property.getValue());
crawl(subUrl, cache, input.get("file.class"));
}
catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}]: {}", property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
catch (IOException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}]: {}", property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
}
}
}
/**
* Crawl.
*
* @param configurationFile
* the input
* @throws StatoolInfosException
* the statool infos exception
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void crawl(final File configurationFile) throws StatoolInfosException, IOException
{
logger.info("Crawl {}", configurationFile.getAbsolutePath());
Configuration configuration = Factory.loadConfiguration(configurationFile);
crawl(configuration);
} }
/** /**
@ -162,42 +104,111 @@ public class Crawler
* @throws IOException * @throws IOException
* Signals that an I/O exception has occurred. * Signals that an I/O exception has occurred.
*/ */
public static void crawl(final URL url, final CrawlCache cache, final String parentFileClass) throws StatoolInfosException, IOException public void crawl(final URL url, final PropertyClassType parent)
{ {
logger.info("Crawling " + url); logger.info("Crawling {}", url);
// Crawl. try
File file = cache.store(url); {
if (file != null) File downloadFile;
try
{
downloadFile = download(url);
}
catch (java.net.ConnectException exception)
{
logger.error("ERROR: crawl failed for [{}]: {}", url.toString(), exception.getMessage());
this.journal.add(url, CrawlStatus.CONNECTERROR);
downloadFile = null;
exception.printStackTrace();
}
catch (FileNotFoundException exception)
{
logger.error("ERROR: crawl failed for [{}]: {}", url.toString(), exception.getMessage());
this.journal.add(url, CrawlStatus.URLNOTFOUND);
downloadFile = null;
exception.printStackTrace();
}
catch (IOException exception)
{
logger.error("ERROR: crawl failed for [{}]: {}", url.toString(), exception.getMessage());
this.journal.add(url, CrawlStatus.DOWNLOADERROR);
downloadFile = null;
exception.printStackTrace();
}
if (downloadFile != null)
{
if (!downloadFile.exists())
{
logger.error("ERROR: download missing.");
this.journal.add(url, CrawlStatus.MISSING);
}
else if (downloadFile.length() == 0)
{
logger.error("ERROR: download empty.");
this.journal.add(url, CrawlStatus.EMPTY);
}
else
{
PathProperties downloadProperties = PathPropertyUtils.load(downloadFile);
PropertyClassType downloadClass = PropertyClassType.of(downloadProperties.get("file.class"));
if ((downloadClass == null) || (!downloadClass.isChildOf(parent)))
{
logger.error("ERROR: bad child class [{}][{}].", downloadClass, parent);
this.journal.add(url, CrawlStatus.BADCHILDCLASS);
}
else
{
File storedFile = this.cache.restoreFile(url);
String storedSha;
if (storedFile == null)
{
storedSha = null;
}
else
{
PathProperties storedProperties = PathPropertyUtils.load(storedFile);
storedSha = storedProperties.get("crawl.file.sha1");
}
String downloadSha = StatoolInfosUtils.sha1sum(downloadFile);
if (StringUtils.equals(downloadSha, storedSha))
{
this.journal.add(url, CrawlStatus.SUCCESS);
}
else
{ {
// Build crawl data. // Build crawl data.
PathProperties crawlSection = new PathPropertyList(); PathProperties crawlSection = new PathPropertyList();
crawlSection.put("crawl.crawler", "StatoolInfos"); crawlSection.put("crawl.crawler", "StatoolInfos");
crawlSection.put("crawl.datetime", LocalDateTime.now().toString()); crawlSection.put("crawl.datetime", LocalDateTime.now().format(DateTimeFormatter.ofPattern("YYYY-MM-dd'T'HH:mm:ss")));
crawlSection.put("crawl.url", url.toString()); crawlSection.put("crawl.url", url.toString());
crawlSection.put("crawl.file.size", FileUtils.sizeOf(file)); crawlSection.put("crawl.file.size", FileUtils.sizeOf(downloadFile));
crawlSection.put("crawl.file.datetime", StatoolInfosUtils.urlLastModified(url).toString()); crawlSection.put("crawl.file.datetime", StatoolInfosUtils.urlLastModified(url).toString());
crawlSection.put("crawl.file.sha1", DigestUtils.sha1Hex(FileUtils.readFileToByteArray(file))); crawlSection.put("crawl.file.sha1", downloadSha);
String crawlSectionLines = crawlSection.toStringListFormatted().toStringSeparatedBy('\n');
// Add crawl data in crawled file. // Add crawl data in crawled file.
String lines = crawlSection.toStringListFormatted().toStringSeparatedBy('\n'); String downloadExtendedLines = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8) + "\n" + crawlSectionLines;
FileUtils.write(file, FileUtils.readFileToString(file, StandardCharsets.UTF_8) + "\n" + lines, StandardCharsets.UTF_8); FileUtils.write(downloadFile, downloadExtendedLines, StandardCharsets.UTF_8);
// Crawl another resources. // Store in cache.
PathProperties properties = PathPropertyUtils.load(file); this.cache.store(url, downloadFile);
downloadFile.delete();
cache.storeQuietly(properties.getURL("organization.logo")); //
cache.storeQuietly(properties.getURL("service.logo")); this.journal.add(url, CrawlStatus.UPDATED);
// Crawl subs.
String fileClass = properties.get("file.class");
if (StringUtils.equalsIgnoreCase(fileClass, parentFileClass))
{
logger.warn("WARNING: file class same than parent for [{}]", url);
} }
else
{ // Cache another resources.
PathProperties subs = properties.getByPrefix("subs"); crawlLogo(downloadProperties.getURL("federation.logo"));
crawlLogo(downloadProperties.getURL("organization.logo"));
crawlLogo(downloadProperties.getURL("service.logo"));
// Do subs.
PathProperties subs = downloadProperties.getByPrefix("subs");
for (PathProperty property : subs) for (PathProperty property : subs)
{ {
if (StringUtils.isNotBlank(property.getValue())) if (StringUtils.isNotBlank(property.getValue()))
@ -205,21 +216,175 @@ public class Crawler
try try
{ {
URL subUrl = new URL(property.getValue()); URL subUrl = new URL(property.getValue());
crawl(subUrl, cache, fileClass); crawl(subUrl, downloadClass);
} }
catch (java.net.MalformedURLException exception) catch (java.net.MalformedURLException exception)
{ {
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage()); logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
this.journal.add(url, CrawlStatus.BADURLFORMAT);
exception.printStackTrace(); exception.printStackTrace();
} }
catch (java.net.ConnectException | FileNotFoundException exception) }
}
}
}
}
}
catch (IOException exception)
{
this.journal.add(url, CrawlStatus.IOERROR);
}
}
/**
* Crawl logo.
*
* @param url
* the url
* @return the file
*/
public File crawlLogo(final URL url)
{
File result;
try
{
if ((url == null) || (!StringUtils.startsWithIgnoreCase(url.getProtocol(), "http")))
{
result = null;
}
else
{
logger.info("Crawling {}", url);
File logoFile;
try
{
logoFile = download(url);
}
catch (java.net.ConnectException exception)
{
logger.error("ERROR: crawl failed (1) for [{}]: {}", url.toString(), exception.getMessage());
this.journal.add(url, CrawlStatus.CONNECTERROR);
logoFile = null;
}
catch (FileNotFoundException exception)
{
logger.error("ERROR: crawl failed (2) for [{}]: {}", url.toString(), exception.getMessage());
this.journal.add(url, CrawlStatus.URLNOTFOUND);
logoFile = null;
}
catch (IOException exception)
{
logger.error("ERROR: crawl failed (3) for [{}]: {}", url.toString(), exception.getMessage());
this.journal.add(url, CrawlStatus.DOWNLOADERROR);
logoFile = null;
}
if (logoFile == null)
{
result = null;
}
else
{
result = this.cache.store(url, logoFile);
this.journal.add(url, CrawlStatus.SUCCESS);
logoFile.delete();
}
}
}
catch (IOException exception)
{
logger.info("Store failed for {}: {}", url, exception.getMessage());
result = null;
}
//
return result;
}
/**
* Download.
*
* @param url
* the url
* @return the file
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public File download(final URL url) throws IOException
{
File result;
if (!StringUtils.startsWith(url.getProtocol(), "http"))
{
logger.warn("WARNING: crawl failed because bad http+ protocol for [{}]", url);
result = null;
}
else
{
final int TIMEOUT = 5000;
result = Files.createTempFile("tmp-", ".statoolsinfos").toFile();
FileUtils.copyURLToFile(url, result, TIMEOUT, TIMEOUT);
}
//
return result;
}
/**
* Restore journal.
*
* @return the crawl journal
* @throws IOException
*/
public CrawlJournal restoreJournal() throws IOException
{
CrawlJournal result;
logger.info("Restoring crawl journal.");
File journalFile = this.cache.restoreFile(getJournalURL());
result = CrawlJournalFile.load(journalFile);
//
return result;
}
/**
* Store journal.
*/
public void storeJournal()
{
try
{
logger.info("Storing crawl journal.");
File file = Files.createTempFile("tmp-", ".statoolsinfos").toFile();
CrawlJournalFile.save(file, this.journal);
this.cache.store(getJournalURL(), file);
file.delete();
}
catch (IOException exception)
{ {
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace(); exception.printStackTrace();
} }
} }
}
} /**
} * Gets the journal URL.
*
* @return the journal URL
* @throws MalformedURLException
*/
public static URL getJournalURL() throws MalformedURLException
{
URL result;
result = new URL("http://localhost/crawl.journal");
//
return result;
} }
} }

122
src/fr/devinsy/statoolinfos/htmlize/CrawlJournalPage.java Normal file
View file

@ -0,0 +1,122 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.statoolinfos.htmlize;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import fr.devinsy.statoolinfos.HtmlizerContext;
import fr.devinsy.statoolinfos.core.Federation;
import fr.devinsy.statoolinfos.core.StatoolInfosException;
import fr.devinsy.statoolinfos.crawl.CrawlCache;
import fr.devinsy.statoolinfos.crawl.CrawlJournal;
import fr.devinsy.statoolinfos.crawl.CrawlLog;
import fr.devinsy.xidyn.XidynException;
import fr.devinsy.xidyn.data.TagDataManager;
import fr.devinsy.xidyn.presenters.PresenterUtils;
/**
* The Class CrawlJournalPage.
*/
public class CrawlJournalPage
{
private static Logger logger = LoggerFactory.getLogger(CrawlJournalPage.class);
/**
* Builds the all.
*
* @throws StatoolInfosException
* @throws IOException
*/
public static void buildAll() throws StatoolInfosException, IOException
{
Federation federation = HtmlizerContext.instance().getFederation();
CrawlCache cache = HtmlizerContext.instance().getCache();
File htmlizeDirectory = HtmlizerContext.instance().getHtmlizeDirectory();
logger.info("Htmlize Crawl Journal pages.");
CrawlJournal journal = HtmlizerContext.instance().getCrawlJournal();
String page = htmlize("Journal des téléchargements", journal);
FileUtils.write(new File(htmlizeDirectory, federation.getTechnicalName() + "-crawl.xhtml"), page, StandardCharsets.UTF_8);
}
/**
* Htmlize.
*
* @param title
* the title
* @param journal
* the journal
* @return the string
* @throws StatoolInfosException
* the statool infos exception
*/
public static String htmlize(final String title, final CrawlJournal journal) throws StatoolInfosException
{
String result;
try
{
logger.debug("Building Crawl journal page…");
TagDataManager data = new TagDataManager();
data.setEscapedContent("title", title);
data.setContent("date", journal.getDatetime().toString());
data.setContent("totalCount", journal.size());
data.setContent("errorCount", journal.getErrors().size());
int index = 0;
for (CrawlLog log : journal)
{
data.setEscapedContent("crawlLogLine", index, "crawlLogLineUrlLink", log.getUrl().toString());
data.setEscapedAttribute("crawlLogLine", index, "crawlLogLineUrlLink", "href", log.getUrl().toString());
data.setContent("crawlLogLine", index, "crawlLogLineStatus", log.getStatus().toString());
if (log.getStatus().isError())
{
data.setAttribute("crawlLogLine", index, "crawlLogLineStatus", "style", "background-color: red;");
}
else
{
data.setAttribute("crawlLogLine", index, "crawlLogLineStatus", "style", "background-color: lime;");
}
index += 1;
}
String content = PresenterUtils.dynamize("/fr/devinsy/statoolinfos/htmlize/crawlJournal.xhtml", data).toString();
BreadcrumbTrail trail = new BreadcrumbTrail();
result = WebCharterView.build(content, trail);
}
catch (XidynException exception)
{
throw new StatoolInfosException("Error building crawl journal page: " + exception.getMessage(), exception);
}
//
return result;
}
}

12
src/fr/devinsy/statoolinfos/htmlize/FederationPage.java
View file

@ -107,9 +107,19 @@ public class FederationPage
data.setAttribute("rawLink", "href", federation.getTechnicalName() + ".properties"); data.setAttribute("rawLink", "href", federation.getTechnicalName() + ".properties");
data.setAttribute("rawCheckLink", "href", federation.getTechnicalName() + "-check.xhtml"); data.setAttribute("rawCheckLink", "href", federation.getTechnicalName() + "-check.xhtml");
data.setAttribute("statsLink", "href", federation.getTechnicalName() + "-stats.xhtml"); data.setAttribute("statsLink", "href", federation.getTechnicalName() + "-stats.xhtml");
data.setAttribute("crawlLink", "href", federation.getTechnicalName() + "-crawl.xhtml");
if (HtmlizerContext.instance().getCrawlJournal().getErrors().isEmpty())
{
data.setAttribute("crawlLinkImg", "src", "circle-icons/download-mono.svg");
}
else
{
data.setAttribute("crawlLinkImg", "src", "circle-icons/download.svg");
}
{ {
PropertyChecks checks = federation.getInputChecksAll(); PropertyChecks checks = federation.getInputChecksAll();

1
src/fr/devinsy/statoolinfos/htmlize/Htmlizer.java
View file

@ -193,6 +193,7 @@ public class Htmlizer
AboutPage.build(); AboutPage.build();
CategoriesPage.build(); CategoriesPage.build();
CategoryPage.buildAll(); CategoryPage.buildAll();
CrawlJournalPage.buildAll();
EditoPage.build(); EditoPage.build();
ExportsPage.build(); ExportsPage.build();
FederationPage.build(); FederationPage.build();

56
src/fr/devinsy/statoolinfos/htmlize/crawlJournal.xhtml Normal file
View file

@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>StatoolInfos</title>
<meta charset="UTF-8" />
<meta name="keywords" content="statoolinfos,devinsy,federation" />
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon" />
<link rel="stylesheet" type="text/css" href="statoolinfos.css" />
<script src="sorttable.js" />
<script src="Chart.bundle.min.js"></script>
</head>
<body>
<div class="center_table" style="width: 1100px;">
<div class="center">
<h2 id="title">Journal des téléchargements</h2>
<div>Nombre de téléchargements : <span id="totalCount">n/a</span></div>
<div>Nombre d'erreurs : <span id="errorCount">n/a</span></div>
<div>Date : <span id="date">n/a</span></div>
</div>
<br/>
<div class="center_table" style="width: 900px;">
<table id="crawlLogs" class="table_classic left">
<thead>
<tr>
<th>URL</th>
<th style="width: 200px;">Statut</th>
</tr>
</thead>
<tbody>
<tr id="crawlLogLine">
<td id="crawlLogLineUrl"><a href="#" id="crawlLogLineUrlLink">n/a</a></td>
<td id="crawlLogLineStatus" class="td_center center">n/a</td>
</tr>
</tbody>
</table>
</div>
</div>
<script type="text/javascript">
$('#crawlLogs').DataTable(
{
paging: false,
ordering: true,
"order": [[ 0, "asc" ]],
language: dataTableFrench,
column:
[ {},{},
{
"bSortable": false
},
]
});
</script>
</body>
</html>

1
src/fr/devinsy/statoolinfos/htmlize/federation.xhtml
View file

@ -31,6 +31,7 @@
<a id="technicalDocLink" href="#"><img id="technicalDocLinkImg" src="circle-icons/tools.svg" class="disabled" title="Documentation technique"/></a> <a id="technicalDocLink" href="#"><img id="technicalDocLinkImg" src="circle-icons/tools.svg" class="disabled" title="Documentation technique"/></a>
<a id="rawCheckLink" href="#"><img id="rawCheckLinkImg" src="circle-icons/clipboard-mono.svg" title="Fichier propriétés analysé"/></a> <a id="rawCheckLink" href="#"><img id="rawCheckLinkImg" src="circle-icons/clipboard-mono.svg" title="Fichier propriétés analysé"/></a>
<a id="rawLink" href="#"><img id="rawLinkImg" src="circle-icons/document-mono.svg" title="Fichier propriétés"/></a> <a id="rawLink" href="#"><img id="rawLinkImg" src="circle-icons/document-mono.svg" title="Fichier propriétés"/></a>
<a id="crawlLink" href="#"><img id="crawlLinkImg" src="circle-icons/download-mono.svg" title="Statut des téléchargements"/></a>
<a id="statsLink" href="#"><img id="statsLinkImg" src="circle-icons/barchart-mono.svg" title="Statistiques"/></a> <a id="statsLink" href="#"><img id="statsLinkImg" src="circle-icons/barchart-mono.svg" title="Statistiques"/></a>
<div style="display: inline-block; vertical-align: middle; font-size: smaller; margin-left: 2px; width: 35px;"> <div style="display: inline-block; vertical-align: middle; font-size: smaller; margin-left: 2px; width: 35px;">
<a id="alertLink" href="#" style="text-decoration: none;"> <a id="alertLink" href="#" style="text-decoration: none;">

81
src/fr/devinsy/statoolinfos/htmlize/stuff/circle-icons/download-ko.svg Normal file
View file

@ -0,0 +1,81 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 85.333336 85.333336"
height="85.333336"
width="85.333336"
xml:space="preserve"
id="svg2"
version="1.1"><metadata
id="metadata8"><rdf:RDF><cc:Work
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
id="defs6"><clipPath
id="clipPath18"
clipPathUnits="userSpaceOnUse"><path
id="path16"
d="M 0,64 H 64 V 0 H 0 Z" /></clipPath><clipPath
id="clipPath30"
clipPathUnits="userSpaceOnUse"><path
id="path28"
d="M 24,52.9823 H 40 V 52.8209 H 24 Z" /></clipPath><clipPath
id="clipPath50"
clipPathUnits="userSpaceOnUse"><path
id="path48"
d="M 8,46.7566 H 56 V 16 H 8 Z" /></clipPath></defs><g
transform="matrix(1.3333333,0,0,-1.3333333,0,85.333333)"
id="g10"><g
id="g12"><g
clip-path="url(#clipPath18)"
id="g14"><g
transform="translate(64,32)"
id="g20"><path
id="path22"
style="fill:#c75c5c;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 0,0 c 0,-17.673 -14.327,-32 -32,-32 -17.673,0 -32,14.327 -32,32 0,17.673 14.327,32 32,32 C -14.327,32 0,17.673 0,0" /></g><g
id="g24"><g
id="g26" /><g
id="g42"><g
style="opacity:0.19999701"
id="g40"
clip-path="url(#clipPath30)"><g
id="g34"
transform="translate(39.9836,52.8257)"><path
id="path32"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0.002,0.017 0.016,0.043 0.016,0.058 0.016,0.219 0.011,0.17 0,0" /></g><g
id="g38"
transform="translate(24,52.8834)"><path
id="path36"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,-0.016 0.015,-0.044 0.018,-0.062 0.006,0.112 0,0.163 0,0" /></g></g></g></g><g
id="g44"><g
id="g46" /><g
id="g58"><g
style="opacity:0.19999701"
id="g56"
clip-path="url(#clipPath50)"><g
id="g54"
transform="translate(48,32)"><path
id="path52"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,6.469 -3.847,12.027 -9.37,14.549 -11.096,15.273 -12,13.954 -12,12.883 v -8.5 C -12,3.555 -11.328,3 -10.5,3 h 3.061 c 1.472,0 1.883,-1.127 0.914,-2.362 l -7.713,-9.829 c -0.969,-1.234 -2.555,-1.234 -3.524,0 l -7.713,9.829 C -26.444,1.873 -26.033,3 -24.561,3 h 3.061 c 0.828,0 1.5,0.555 1.5,1.383 v 8.5 c 0,1.105 -1.504,2.143 -2.753,1.61 -0.002,-10e-4 -0.005,0.002 -0.007,10e-4 -4.311,-2.02 -7.573,-5.905 -8.748,-10.62 C -36.313,3.146 -40,-0.991 -40,-6 c 0,-5.523 4.477,-10 10,-10 H 0 c 4.418,0 8,3.582 8,8 0,4.418 -3.582,8 -8,8" /></g></g></g></g><g
transform="translate(48,34)"
id="g60"><path
id="path62"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,6.469 -3.847,12.027 -9.37,14.549 -11.096,15.273 -12,13.954 -12,12.883 v -8.5 C -12,3.555 -11.328,3 -10.5,3 h 3.061 c 1.472,0 1.883,-1.127 0.914,-2.362 l -7.713,-9.829 c -0.969,-1.234 -2.555,-1.234 -3.524,0 l -7.713,9.829 C -26.444,1.873 -26.033,3 -24.561,3 h 3.061 c 0.828,0 1.5,0.555 1.5,1.383 v 8.5 c 0,1.105 -1.504,2.143 -2.753,1.61 -0.002,-10e-4 -0.005,0.002 -0.007,10e-4 -4.311,-2.02 -7.573,-5.905 -8.748,-10.62 C -36.313,3.146 -40,-0.991 -40,-6 c 0,-5.523 4.477,-10 10,-10 H 0 c 4.418,0 8,3.582 8,8 0,4.418 -3.582,8 -8,8" /></g><g
transform="translate(39.9836,54.8257)"
id="g64"><path
id="path66"
style="fill:#8c81d4;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0.002,0.017 0.016,0.043 0.016,0.058 0.016,0.219 0.011,0.17 0,0" /></g><g
transform="translate(24,54.8834)"
id="g68"><path
id="path70"
style="fill:#8c81d4;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,-0.016 0.015,-0.044 0.018,-0.062 0.006,0.112 0,0.163 0,0" /></g></g></g></g></svg>
Side by side

After

Width:  |  Height:  |  Size: 4.6 KiB

46
src/fr/devinsy/statoolinfos/htmlize/stuff/circle-icons/download-mono.svg Normal file
View file

@ -0,0 +1,46 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 85.333336 85.333336"
height="85.333336"
width="85.333336"
xml:space="preserve"
id="svg2"
version="1.1"><metadata
id="metadata8"><rdf:RDF><cc:Work
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
id="defs6"><clipPath
id="clipPath18"
clipPathUnits="userSpaceOnUse"><path
id="path16"
d="M 0,64 H 64 V 0 H 0 Z" /></clipPath></defs><g
transform="matrix(1.3333333,0,0,-1.3333333,0,85.333333)"
id="g10"><g
id="g12"><g
clip-path="url(#clipPath18)"
id="g14"><g
transform="translate(48,34)"
id="g20"><path
id="path22"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,6.469 -3.847,12.027 -9.37,14.549 -11.096,15.273 -12,13.954 -12,12.883 v -8.5 C -12,3.555 -11.328,3 -10.5,3 h 3.061 c 1.472,0 1.883,-1.127 0.914,-2.362 l -7.713,-9.829 c -0.969,-1.234 -2.555,-1.234 -3.524,0 l -7.713,9.829 C -26.444,1.873 -26.033,3 -24.561,3 h 3.061 c 0.828,0 1.5,0.555 1.5,1.383 v 8.5 c 0,1.105 -1.504,2.143 -2.753,1.61 -0.002,-10e-4 -0.005,0.002 -0.007,10e-4 -4.311,-2.02 -7.573,-5.905 -8.748,-10.62 C -36.313,3.146 -40,-0.991 -40,-6 c 0,-5.523 4.477,-10 10,-10 H 0 c 4.418,0 8,3.582 8,8 0,4.418 -3.582,8 -8,8" /></g><g
transform="translate(24,54.8834)"
id="g24"><path
id="path26"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,-0.016 0.015,-0.044 0.018,-0.062 0.006,0.112 0,0.163 0,0" /></g><g
transform="translate(39.9836,54.8257)"
id="g28"><path
id="path30"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0.002,0.017 0.016,0.043 0.016,0.058 0.016,0.219 0.011,0.17 0,0" /></g><g
transform="translate(48,18)"
id="g32"><path
id="path34"
style="fill:#4f5d73;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 0,0 h -30 c -5.523,0 -10,4.477 -10,10 0,5.009 3.687,9.146 8.492,9.874 1.175,4.715 4.437,8.6 8.748,10.62 0.002,0.001 0.005,-0.002 0.007,-0.001 1.249,0.533 2.753,-0.505 2.753,-1.61 v -8.5 C -20,19.555 -20.672,19 -21.5,19 h -3.061 c -1.472,0 -1.883,-1.127 -0.914,-2.362 l 7.713,-9.829 c 0.969,-1.234 2.555,-1.234 3.524,0 l 7.713,9.829 C -5.556,17.873 -5.967,19 -7.439,19 H -10.5 c -0.828,0 -1.5,0.555 -1.5,1.383 v 8.5 c 0,1.071 0.904,2.39 2.63,1.666 C -3.847,28.027 0,22.469 0,16 4.418,16 8,12.418 8,8 8,3.582 4.418,0 0,0 m -23.982,36.821 c -0.003,0.018 -0.018,0.047 -0.018,0.062 0,0.163 0.006,0.112 0.018,-0.062 M -8,36.883 C -8,36.869 -8.014,36.842 -8.016,36.826 -8.005,36.996 -8,37.044 -8,36.883 M -16,46 c -17.673,0 -32,-14.327 -32,-32 0,-17.673 14.327,-32 32,-32 17.673,0 32,14.327 32,32 0,17.673 -14.327,32 -32,32" /></g></g></g></g></svg>
Side by side

After

Width:  |  Height:  |  Size: 3.2 KiB

110
src/fr/devinsy/statoolinfos/htmlize/stuff/circle-icons/download-ok.svg Normal file
View file

@ -0,0 +1,110 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
viewBox="0 0 85.333336 85.333336"
height="85.333336"
width="85.333336"
xml:space="preserve"
id="svg2"
version="1.1"
sodipodi:docname="download-red.svg"
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"><sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1388"
inkscape:window-height="998"
id="namedview36"
showgrid="false"
inkscape:zoom="8.2265626"
inkscape:cx="61.943276"
inkscape:cy="42.666669"
inkscape:window-x="438"
inkscape:window-y="25"
inkscape:window-maximized="0"
inkscape:current-layer="svg2" /><metadata
id="metadata8"><rdf:RDF><cc:Work
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
id="defs6"><clipPath
id="clipPath18"
clipPathUnits="userSpaceOnUse"><path
id="path16"
d="M 0,64 H 64 V 0 H 0 Z" /></clipPath><clipPath
id="clipPath30"
clipPathUnits="userSpaceOnUse"><path
id="path28"
d="M 24,52.9823 H 40 V 52.8209 H 24 Z" /></clipPath><clipPath
id="clipPath50"
clipPathUnits="userSpaceOnUse"><path
id="path48"
d="M 8,46.7566 H 56 V 16 H 8 Z" /></clipPath></defs><g
id="g271"><g
id="g245"><g
id="g220"><g
id="g196"><path
id="path22-3"
style="fill:#76c2af;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:1.33333325"
d="m 85.333331,42.666667 c 0,23.563999 -19.102666,42.666665 -42.666665,42.666665 C 19.102666,85.333332 -2e-7,66.230666 -2e-7,42.666667 -2e-7,19.102667 19.102666,10e-7 42.666666,10e-7 c 23.563999,0 42.666665,19.102666 42.666665,42.666666"
inkscape:connector-curvature="0" /><g
id="g24"
transform="matrix(1.3333333,0,0,-1.3333333,0,85.333333)"><g
id="g26" /><g
id="g42"><g
clip-path="url(#clipPath30)"
id="g40"
style="opacity:0.19999701"><g
transform="translate(39.9836,52.8257)"
id="g34"><path
inkscape:connector-curvature="0"
d="M 0,0 C 0.002,0.017 0.016,0.043 0.016,0.058 0.016,0.219 0.011,0.17 0,0"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path32" /></g><g
transform="translate(24,52.8834)"
id="g38"><path
inkscape:connector-curvature="0"
d="M 0,0 C 0,-0.016 0.015,-0.044 0.018,-0.062 0.006,0.112 0,0.163 0,0"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path36" /></g></g></g></g><g
id="g44"
transform="matrix(1.3333333,0,0,-1.3333333,0,85.333333)"><g
id="g46" /><g
id="g58"><g
clip-path="url(#clipPath50)"
id="g56"
style="opacity:0.19999701"><g
transform="translate(48,32)"
id="g54"><path
inkscape:connector-curvature="0"
d="M 0,0 C 0,6.469 -3.847,12.027 -9.37,14.549 -11.096,15.273 -12,13.954 -12,12.883 v -8.5 C -12,3.555 -11.328,3 -10.5,3 h 3.061 c 1.472,0 1.883,-1.127 0.914,-2.362 l -7.713,-9.829 c -0.969,-1.234 -2.555,-1.234 -3.524,0 l -7.713,9.829 C -26.444,1.873 -26.033,3 -24.561,3 h 3.061 c 0.828,0 1.5,0.555 1.5,1.383 v 8.5 c 0,1.105 -1.504,2.143 -2.753,1.61 -0.002,-10e-4 -0.005,0.002 -0.007,10e-4 -4.311,-2.02 -7.573,-5.905 -8.748,-10.62 C -36.313,3.146 -40,-0.991 -40,-6 c 0,-5.523 4.477,-10 10,-10 H 0 c 4.418,0 8,3.582 8,8 0,4.418 -3.582,8 -8,8"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path52" /></g></g></g></g><g
id="g60"
transform="matrix(1.3333333,0,0,-1.3333333,63.999998,40.000001)"><path
inkscape:connector-curvature="0"
d="M 0,0 C 0,6.469 -3.847,12.027 -9.37,14.549 -11.096,15.273 -12,13.954 -12,12.883 v -8.5 C -12,3.555 -11.328,3 -10.5,3 h 3.061 c 1.472,0 1.883,-1.127 0.914,-2.362 l -7.713,-9.829 c -0.969,-1.234 -2.555,-1.234 -3.524,0 l -7.713,9.829 C -26.444,1.873 -26.033,3 -24.561,3 h 3.061 c 0.828,0 1.5,0.555 1.5,1.383 v 8.5 c 0,1.105 -1.504,2.143 -2.753,1.61 -0.002,-10e-4 -0.005,0.002 -0.007,10e-4 -4.311,-2.02 -7.573,-5.905 -8.748,-10.62 C -36.313,3.146 -40,-0.991 -40,-6 c 0,-5.523 4.477,-10 10,-10 H 0 c 4.418,0 8,3.582 8,8 0,4.418 -3.582,8 -8,8"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path62" /></g><g
id="g64"
transform="matrix(1.3333333,0,0,-1.3333333,53.311465,12.232401)"><path
inkscape:connector-curvature="0"
d="M 0,0 C 0.002,0.017 0.016,0.043 0.016,0.058 0.016,0.219 0.011,0.17 0,0"
style="fill:#8c81d4;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path66" /></g><g
id="g68"
transform="matrix(1.3333333,0,0,-1.3333333,31.999999,12.155468)"><path
inkscape:connector-curvature="0"
d="M 0,0 C 0,-0.016 0.015,-0.044 0.018,-0.062 0.006,0.112 0,0.163 0,0"
style="fill:#8c81d4;fill-opacity:1;fill-rule:nonzero;stroke:none"
id="path70" /></g></g></g></g></g></svg>
Side by side

After

Width:  |  Height:  |  Size: 6 KiB

81
src/fr/devinsy/statoolinfos/htmlize/stuff/circle-icons/download.svg Normal file
View file

@ -0,0 +1,81 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 85.333336 85.333336"
height="85.333336"
width="85.333336"
xml:space="preserve"
id="svg2"
version="1.1"><metadata
id="metadata8"><rdf:RDF><cc:Work
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
id="defs6"><clipPath
id="clipPath18"
clipPathUnits="userSpaceOnUse"><path
id="path16"
d="M 0,64 H 64 V 0 H 0 Z" /></clipPath><clipPath
id="clipPath30"
clipPathUnits="userSpaceOnUse"><path
id="path28"
d="M 24,52.9823 H 40 V 52.8209 H 24 Z" /></clipPath><clipPath
id="clipPath50"
clipPathUnits="userSpaceOnUse"><path
id="path48"
d="M 8,46.7566 H 56 V 16 H 8 Z" /></clipPath></defs><g
transform="matrix(1.3333333,0,0,-1.3333333,0,85.333333)"
id="g10"><g
id="g12"><g
clip-path="url(#clipPath18)"
id="g14"><g
transform="translate(64,32)"
id="g20"><path
id="path22"
style="fill:#c75c5c;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="m 0,0 c 0,-17.673 -14.327,-32 -32,-32 -17.673,0 -32,14.327 -32,32 0,17.673 14.327,32 32,32 C -14.327,32 0,17.673 0,0" /></g><g
id="g24"><g
id="g26" /><g
id="g42"><g
style="opacity:0.19999701"
id="g40"
clip-path="url(#clipPath30)"><g
id="g34"
transform="translate(39.9836,52.8257)"><path
id="path32"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0.002,0.017 0.016,0.043 0.016,0.058 0.016,0.219 0.011,0.17 0,0" /></g><g
id="g38"
transform="translate(24,52.8834)"><path
id="path36"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,-0.016 0.015,-0.044 0.018,-0.062 0.006,0.112 0,0.163 0,0" /></g></g></g></g><g
id="g44"><g
id="g46" /><g
id="g58"><g
style="opacity:0.19999701"
id="g56"
clip-path="url(#clipPath50)"><g
id="g54"
transform="translate(48,32)"><path
id="path52"
style="fill:#231f20;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,6.469 -3.847,12.027 -9.37,14.549 -11.096,15.273 -12,13.954 -12,12.883 v -8.5 C -12,3.555 -11.328,3 -10.5,3 h 3.061 c 1.472,0 1.883,-1.127 0.914,-2.362 l -7.713,-9.829 c -0.969,-1.234 -2.555,-1.234 -3.524,0 l -7.713,9.829 C -26.444,1.873 -26.033,3 -24.561,3 h 3.061 c 0.828,0 1.5,0.555 1.5,1.383 v 8.5 c 0,1.105 -1.504,2.143 -2.753,1.61 -0.002,-10e-4 -0.005,0.002 -0.007,10e-4 -4.311,-2.02 -7.573,-5.905 -8.748,-10.62 C -36.313,3.146 -40,-0.991 -40,-6 c 0,-5.523 4.477,-10 10,-10 H 0 c 4.418,0 8,3.582 8,8 0,4.418 -3.582,8 -8,8" /></g></g></g></g><g
transform="translate(48,34)"
id="g60"><path
id="path62"
style="fill:#ffffff;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,6.469 -3.847,12.027 -9.37,14.549 -11.096,15.273 -12,13.954 -12,12.883 v -8.5 C -12,3.555 -11.328,3 -10.5,3 h 3.061 c 1.472,0 1.883,-1.127 0.914,-2.362 l -7.713,-9.829 c -0.969,-1.234 -2.555,-1.234 -3.524,0 l -7.713,9.829 C -26.444,1.873 -26.033,3 -24.561,3 h 3.061 c 0.828,0 1.5,0.555 1.5,1.383 v 8.5 c 0,1.105 -1.504,2.143 -2.753,1.61 -0.002,-10e-4 -0.005,0.002 -0.007,10e-4 -4.311,-2.02 -7.573,-5.905 -8.748,-10.62 C -36.313,3.146 -40,-0.991 -40,-6 c 0,-5.523 4.477,-10 10,-10 H 0 c 4.418,0 8,3.582 8,8 0,4.418 -3.582,8 -8,8" /></g><g
transform="translate(39.9836,54.8257)"
id="g64"><path
id="path66"
style="fill:#8c81d4;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0.002,0.017 0.016,0.043 0.016,0.058 0.016,0.219 0.011,0.17 0,0" /></g><g
transform="translate(24,54.8834)"
id="g68"><path
id="path70"
style="fill:#8c81d4;fill-opacity:1;fill-rule:nonzero;stroke:none"
d="M 0,0 C 0,-0.016 0.015,-0.044 0.018,-0.062 0.006,0.112 0,0.163 0,0" /></g></g></g></g></svg>
Side by side

After

Width:  |  Height:  |  Size: 4.6 KiB

2
src/fr/devinsy/statoolinfos/properties/PathProperties.java
View file

@ -35,7 +35,7 @@ public interface PathProperties extends Iterable<PathProperty>
boolean add(PathProperty property); boolean add(PathProperty property);
/** /**
* h Gets the. * Gets the.
* *
* @param path * @param path
* the path * the path

12
src/fr/devinsy/statoolinfos/properties/PathPropertyUtils.java
View file

@ -27,6 +27,8 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.net.URL; import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -43,8 +45,6 @@ public class PathPropertyUtils
{ {
private static final Logger logger = LoggerFactory.getLogger(PathPropertyUtils.class); private static final Logger logger = LoggerFactory.getLogger(PathPropertyUtils.class);
public static final String DEFAULT_CHARSET_NAME = "UTF-8";
/** /**
* Checks if is property line. * Checks if is property line.
* *
@ -82,7 +82,7 @@ public class PathPropertyUtils
{ {
PathProperties result; PathProperties result;
result = load(file, DEFAULT_CHARSET_NAME); result = load(file, StandardCharsets.UTF_8);
// //
return result; return result;
@ -93,13 +93,13 @@ public class PathPropertyUtils
* *
* @param file * @param file
* the file * the file
* @param charsetName * @param charset
* the charset name * the charset name
* @return the path properties * @return the path properties
* @throws IOException * @throws IOException
* Signals that an I/O exception has occurred. * Signals that an I/O exception has occurred.
*/ */
public static PathProperties load(final File file, final String charsetName) throws IOException public static PathProperties load(final File file, final Charset charset) throws IOException
{ {
PathProperties result; PathProperties result;
@ -114,7 +114,7 @@ public class PathPropertyUtils
BufferedReader in = null; BufferedReader in = null;
try try
{ {
in = new BufferedReader(new InputStreamReader(new FileInputStream(file), charsetName)); in = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
result = read(in); result = read(in);
} }
finally finally