statool/statoolinfos
statool/statoolinfos
1
0
Fork 0
Code Issues Pull requests Releases 8 Activity

Improved logs and sub error management.

Browse source
This commit is contained in:
Christian P. MOMON 2021-01-07 09:18:09 +01:00
parent 35bddb9f26
commit a653627030
3 changed files with 60 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/fr/devinsy/statoolinfos/core/Factory.java
View file

@ -173,11 +173,7 @@ public class Factory
{ {
URL inputURL = new URL(property.getValue()); URL inputURL = new URL(property.getValue());
Organization organization = loadOrganization(inputURL, cache); Organization organization = loadOrganization(inputURL, cache);
if (organization == null) if (organization != null)
{
logger.error("Loading organization failed for [{}]", property.getValue());
}
else
{ {
result.getOrganizations().add(organization); result.getOrganizations().add(organization);
} }
@ -243,6 +239,7 @@ public class Factory
if (inputFile == null) if (inputFile == null)
{ {
result = null; result = null;
logger.warn("WARNING: organization not found in cache [{}]", inputURL);
} }
else else
{ {

2
src/fr/devinsy/statoolinfos/crawl/CrawlCache.java
View file

@ -487,7 +487,7 @@ public class CrawlCache
} }
catch (IOException exception) catch (IOException exception)
{ {
logger.info("Store faile for {}: {}", url, exception.getMessage()); logger.info("Store failed for {}: {}", url, exception.getMessage());
result = null; result = null;
} }

93
src/fr/devinsy/statoolinfos/crawl/Crawler.java
View file

@ -110,8 +110,24 @@ public class Crawler
PathProperties subs = input.getByPrefix("subs"); PathProperties subs = input.getByPrefix("subs");
for (PathProperty property : subs) for (PathProperty property : subs)
{ {
URL url = new URL(property.getValue()); if (StringUtils.isNotBlank(property.getValue()))
crawl(url, cache); {
try
{
URL subUrl = new URL(property.getValue());
crawl(subUrl, cache);
}
catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}]: {}", property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
catch (java.net.ConnectException | FileNotFoundException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}]: {}", property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
}
} }
} }
@ -148,47 +164,52 @@ public class Crawler
*/ */
public static void crawl(final URL url, final CrawlCache cache) throws StatoolInfosException, IOException public static void crawl(final URL url, final CrawlCache cache) throws StatoolInfosException, IOException
{ {
try logger.info("Crawling " + url);
// Crawl.
File file = cache.store(url);
// Build crawl data.
PathProperties crawlSection = new PathPropertyList();
crawlSection.put("crawl.crawler", "StatoolInfos");
crawlSection.put("crawl.datetime", LocalDateTime.now().toString());
crawlSection.put("crawl.url", url.toString());
crawlSection.put("crawl.file.size", FileUtils.sizeOf(file));
crawlSection.put("crawl.file.datetime", StatoolInfosUtils.urlLastModified(url).toString());
crawlSection.put("crawl.file.sha1", DigestUtils.sha1Hex(FileUtils.readFileToByteArray(file)));
// Add crawl data in crawled file.
String lines = crawlSection.toStringListFormatted().toStringSeparatedBy('\n');
FileUtils.write(file, FileUtils.readFileToString(file, StandardCharsets.UTF_8) + "\n" + lines, StandardCharsets.UTF_8);
// Crawl another resources.
PathProperties properties = PathPropertyUtils.load(file);
cache.storeQuietly(properties.getURL("organization.logo"));
cache.storeQuietly(properties.getURL("service.logo"));
// Crawl subs.
PathProperties subs = properties.getByPrefix("subs");
for (PathProperty property : subs)
{ {
logger.info("Crawling " + url); if (StringUtils.isNotBlank(property.getValue()))
// Crawl.
File file = cache.store(url);
// Build crawl data.
PathProperties crawlSection = new PathPropertyList();
crawlSection.put("crawl.crawler", "StatoolInfos");
crawlSection.put("crawl.datetime", LocalDateTime.now().toString());
crawlSection.put("crawl.url", url.toString());
crawlSection.put("crawl.file.size", FileUtils.sizeOf(file));
crawlSection.put("crawl.file.datetime", StatoolInfosUtils.urlLastModified(url).toString());
crawlSection.put("crawl.file.sha1", DigestUtils.sha1Hex(FileUtils.readFileToByteArray(file)));
// Add crawl data in crawled file.
String lines = crawlSection.toStringListFormatted().toStringSeparatedBy('\n');
FileUtils.write(file, FileUtils.readFileToString(file, StandardCharsets.UTF_8) + "\n" + lines, StandardCharsets.UTF_8);
// Crawl another resources.
PathProperties properties = PathPropertyUtils.load(file);
cache.storeQuietly(properties.getURL("organization.logo"));
cache.storeQuietly(properties.getURL("service.logo"));
// Crawl subs.
PathProperties subs = properties.getByPrefix("subs");
for (PathProperty property : subs)
{ {
if (StringUtils.isNotBlank(property.getValue())) try
{ {
URL subUrl = new URL(property.getValue()); URL subUrl = new URL(property.getValue());
crawl(subUrl, cache); crawl(subUrl, cache);
} }
catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
catch (java.net.ConnectException | FileNotFoundException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
} }
} }
catch (java.net.ConnectException | FileNotFoundException exception)
{
logger.error("ERROR: crawl failed for [{}]: {}", url.toString(), exception.getMessage());
exception.printStackTrace();
}
} }
} }