statool/statoolinfosweb
statool/statoolinfosweb
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Activity

Fixed sub file class same than parent case in crawling.

Browse source
This commit is contained in:
Christian P. MOMON 2021-01-13 02:20:28 +01:00
parent 51a46a5ed4
commit 040f855647
1 changed files with 27 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
src/fr/devinsy/statoolinfos/crawl/Crawler.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -115,7 +115,7 @@ public class Crawler
try try
{ {
URL subUrl = new URL(property.getValue()); URL subUrl = new URL(property.getValue());
crawl(subUrl, cache); crawl(subUrl, cache, input.get("file.class"));
} }
catch (java.net.MalformedURLException exception) catch (java.net.MalformedURLException exception)
{ {
@ -162,7 +162,7 @@ public class Crawler
* @throws IOException * @throws IOException
* Signals that an I/O exception has occurred. * Signals that an I/O exception has occurred.
*/ */
public static void crawl(final URL url, final CrawlCache cache) throws StatoolInfosException, IOException public static void crawl(final URL url, final CrawlCache cache, final String parentFileClass) throws StatoolInfosException, IOException
{ {
logger.info("Crawling " + url); logger.info("Crawling " + url);
@ -190,25 +190,33 @@ public class Crawler
cache.storeQuietly(properties.getURL("service.logo")); cache.storeQuietly(properties.getURL("service.logo"));
// Crawl subs. // Crawl subs.
PathProperties subs = properties.getByPrefix("subs"); String fileClass = properties.get("file.class");
for (PathProperty property : subs) if (StringUtils.equalsIgnoreCase(fileClass, parentFileClass))
{ {
if (StringUtils.isNotBlank(property.getValue())) logger.warn("WARNING: file class same than parent for [{}]", url);
}
else
{
PathProperties subs = properties.getByPrefix("subs");
for (PathProperty property : subs)
{ {
try if (StringUtils.isNotBlank(property.getValue()))
{ {
URL subUrl = new URL(property.getValue()); try
crawl(subUrl, cache); {
} URL subUrl = new URL(property.getValue());
catch (java.net.MalformedURLException exception) crawl(subUrl, cache, fileClass);
{ }
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage()); catch (java.net.MalformedURLException exception)
exception.printStackTrace(); {
} logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
catch (java.net.ConnectException | FileNotFoundException exception) exception.printStackTrace();
{ }
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage()); catch (java.net.ConnectException | FileNotFoundException exception)
exception.printStackTrace(); {
logger.error("ERROR: subcrawl failed for [{}][{}][{}]: {}", url.toString(), property.getPath(), property.getValue(), exception.getMessage());
exception.printStackTrace();
}
} }
} }
} }