statool/statoolinfosweb
statool/statoolinfosweb
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Activity

Fixed sub file class same than parent case in crawling.

Browse source
This commit is contained in:
Christian P. MOMON 2021-01-13 02:20:28 +01:00
parent 51a46a5ed4
commit 040f855647
1 changed files with 27 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/fr/devinsy/statoolinfos/crawl/Crawler.java
View file

@ -1,5 +1,5 @@
/* /*
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org> * Copyright (C) 2020-2021 Christian Pierre MOMON <christian@momon.org>
* *
* This file is part of StatoolInfos, simple service statistics tool. * This file is part of StatoolInfos, simple service statistics tool.
* *
@ -115,7 +115,7 @@ public class Crawler
try try
{ {
URL subUrl = new URL(property.getValue()); URL subUrl = new URL(property.getValue());
crawl(subUrl, cache); crawl(subUrl, cache, input.get("file.class"));
} }
catch (java.net.MalformedURLException exception) catch (java.net.MalformedURLException exception)
{ {
@ -162,7 +162,7 @@ public class Crawler
* @throws IOException * @throws IOException
* Signals that an I/O exception has occurred. * Signals that an I/O exception has occurred.
*/ */
public static void crawl(final URL url, final CrawlCache cache) throws StatoolInfosException, IOException public static void crawl(final URL url, final CrawlCache cache, final String parentFileClass) throws StatoolInfosException, IOException
{ {
logger.info("Crawling " + url); logger.info("Crawling " + url);
@ -190,6 +190,13 @@ public class Crawler
cache.storeQuietly(properties.getURL("service.logo")); cache.storeQuietly(properties.getURL("service.logo"));
// Crawl subs. // Crawl subs.
String fileClass = properties.get("file.class");
if (StringUtils.equalsIgnoreCase(fileClass, parentFileClass))
{
logger.warn("WARNING: file class same than parent for [{}]", url);
}
else
{
PathProperties subs = properties.getByPrefix("subs"); PathProperties subs = properties.getByPrefix("subs");
for (PathProperty property : subs) for (PathProperty property : subs)
{ {
@ -198,7 +205,7 @@ public class Crawler
try try
{ {
URL subUrl = new URL(property.getValue()); URL subUrl = new URL(property.getValue());
crawl(subUrl, cache); crawl(subUrl, cache, fileClass);
} }
catch (java.net.MalformedURLException exception) catch (java.net.MalformedURLException exception)
{ {
@ -215,3 +222,4 @@ public class Crawler
} }
} }
} }
}