statool/statoolinfos
statool/statoolinfos
1
0
Fork 0
Code Issues Pull requests Releases 8 Activity

Improved logs and sub error management.

Browse source
This commit is contained in:
Christian P. MOMON 2021-01-07 09:18:09 +01:00
parent 35bddb9f26
commit a653627030
3 changed files with 60 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/fr/devinsy/statoolinfos/core/Factory.java
View file

@ -173,11 +173,7 @@ public class Factory
{
URL inputURL = new URL(property.getValue());
Organization organization = loadOrganization(inputURL, cache);
if (organization == null)
{
logger.error("Loading organization failed for [{}]", property.getValue());
}
else
if (organization != null)
{
result.getOrganizations().add(organization);
}
@ -243,6 +239,7 @@ public class Factory
if (inputFile == null)
{
result = null;
logger.warn("WARNING: organization not found in cache [{}]", inputURL);
}
else
{

2
src/fr/devinsy/statoolinfos/crawl/CrawlCache.java
View file

@ -487,7 +487,7 @@ public class CrawlCache
}
catch (IOException exception)
{
logger.info("Store faile for {}: {}", url, exception.getMessage());
logger.info("Store failed for {}: {}", url, exception.getMessage());
result = null;
}

33
src/fr/devinsy/statoolinfos/crawl/Crawler.java
View file

@ -110,8 +110,24 @@ public class Crawler
PathProperties subs = input.getByPrefix("subs");
for (PathProperty property : subs)
{
URL url = new URL(property.getValue());
crawl(url, cache);
if (StringUtils.isNotBlank(property.getValue()))
{
try
{
URL subUrl = new URL(property.getValue());
crawl(subUrl, cache);
}
catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}]: {}", property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
catch (java.net.ConnectException | FileNotFoundException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}]: {}", property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
}
}
}
@ -147,8 +163,6 @@ public class Crawler
* Signals that an I/O exception has occurred.
*/
public static void crawl(final URL url, final CrawlCache cache) throws StatoolInfosException, IOException
{
try
{
logger.info("Crawling " + url);
@ -179,16 +193,23 @@ public class Crawler
for (PathProperty property : subs)
{
if (StringUtils.isNotBlank(property.getValue()))
{
try
{
URL subUrl = new URL(property.getValue());
crawl(subUrl, cache);
}
}
catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
catch (java.net.ConnectException | FileNotFoundException exception)
{
logger.error("ERROR: crawl failed for [{}]: {}", url.toString(), exception.getMessage());
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
}
}
}
}