statool/statoolinfos
statool/statoolinfos
1
0
Fork 0
Code Issues Pull requests Releases 8 Activity

Fixed sub file class same than parent case in crawling.

Browse source
This commit is contained in:
Christian P. MOMON 2021-01-13 02:20:28 +01:00
parent 51a46a5ed4
commit 040f855647
1 changed files with 27 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
src/fr/devinsy/statoolinfos/crawl/Crawler.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
@ -115,7 +115,7 @@ public class Crawler
try
{
URL subUrl = new URL(property.getValue());
crawl(subUrl, cache);
crawl(subUrl, cache, input.get("file.class"));
}
catch (java.net.MalformedURLException exception)
{
@ -162,7 +162,7 @@ public class Crawler
* @throws IOException
* Signals that an I/O exception has occurred.
*/
public static void crawl(final URL url, final CrawlCache cache) throws StatoolInfosException, IOException
public static void crawl(final URL url, final CrawlCache cache, final String parentFileClass) throws StatoolInfosException, IOException
{
logger.info("Crawling " + url);
@ -190,25 +190,33 @@ public class Crawler
cache.storeQuietly(properties.getURL("service.logo"));
// Crawl subs.
PathProperties subs = properties.getByPrefix("subs");
for (PathProperty property : subs)
String fileClass = properties.get("file.class");
if (StringUtils.equalsIgnoreCase(fileClass, parentFileClass))
{
if (StringUtils.isNotBlank(property.getValue()))
logger.warn("WARNING: file class same than parent for [{}]", url);
}
else
{
PathProperties subs = properties.getByPrefix("subs");
for (PathProperty property : subs)
{
try
if (StringUtils.isNotBlank(property.getValue()))
{
URL subUrl = new URL(property.getValue());
crawl(subUrl, cache);
}
catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
catch (java.net.ConnectException | FileNotFoundException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
try
{
URL subUrl = new URL(property.getValue());
crawl(subUrl, cache, fileClass);
}
catch (java.net.MalformedURLException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
catch (java.net.ConnectException | FileNotFoundException exception)
{
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
}
}
}