statool/statoolinfos
statool/statoolinfos
1
0
Fork 0
Code Issues Pull requests Releases 8 Activity

Improved logs and sub error management.

Browse source
This commit is contained in:
Christian P. MOMON 2021-01-07 09:18:09 +01:00
parent 35bddb9f26
commit a653627030
3 changed files with 60 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/fr/devinsy/statoolinfos/core/Factory.java
View file

@ -173,11 +173,7 @@ public class Factory
{ {
URL inputURL = new URL(property.getValue()); URL inputURL = new URL(property.getValue());
Organization organization = loadOrganization(inputURL, cache); Organization organization = loadOrganization(inputURL, cache);
if (organization == null) if (organization != null)
{
logger.error("Loading organization failed for [{}]", property.getValue());
}
else
{ {
result.getOrganizations().add(organization); result.getOrganizations().add(organization);
} }
@ -243,6 +239,7 @@ public class Factory
if (inputFile == null) if (inputFile == null)
{ {
result = null; result = null;
logger.warn("WARNING: organization not found in cache [{}]", inputURL);
} }
else else
{ {

2
src/fr/devinsy/statoolinfos/crawl/CrawlCache.java
View file

@ -487,7 +487,7 @@ public class CrawlCache
} }
catch (IOException exception) catch (IOException exception)
{ {
logger.info("Store faile for {}: {}", url, exception.getMessage()); logger.info("Store failed for {}: {}", url, exception.getMessage());
result = null; result = null;
} }

33
src/fr/devinsy/statoolinfos/crawl/Crawler.java
View file

@ -110,8 +110,24 @@ public class Crawler
PathProperties subs = input.getByPrefix("subs"); PathProperties subs = input.getByPrefix("subs");
for (PathProperty property : subs) for (PathProperty property : subs)
{ {
URL url = new URL(property.getValue()); if (StringUtils.isNotBlank(property.getValue()))
crawl(url, cache); {
try
{
URL subUrl = new URL(property.getValue());
crawl(subUrl, cache);
}
catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}]: {}", property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
catch (java.net.ConnectException | FileNotFoundException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}]: {}", property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
}
} }
} }
@ -147,8 +163,6 @@ public class Crawler
* Signals that an I/O exception has occurred. * Signals that an I/O exception has occurred.
*/ */
public static void crawl(final URL url, final CrawlCache cache) throws StatoolInfosException, IOException public static void crawl(final URL url, final CrawlCache cache) throws StatoolInfosException, IOException
{
try
{ {
logger.info("Crawling " + url); logger.info("Crawling " + url);
@ -179,16 +193,23 @@ public class Crawler
for (PathProperty property : subs) for (PathProperty property : subs)
{ {
if (StringUtils.isNotBlank(property.getValue())) if (StringUtils.isNotBlank(property.getValue()))
{
try
{ {
URL subUrl = new URL(property.getValue()); URL subUrl = new URL(property.getValue());
crawl(subUrl, cache); crawl(subUrl, cache);
} }
} catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
} }
catch (java.net.ConnectException | FileNotFoundException exception) catch (java.net.ConnectException | FileNotFoundException exception)
{ {
logger.error("ERROR: crawl failed for [{}]: {}", url.toString(), exception.getMessage()); logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace(); exception.printStackTrace();
} }
} }
}
}
} }