333 lines
8.7 KiB
Java
333 lines
8.7 KiB
Java
/*
|
|
* Copyright (C) 2020 Christian Pierre MOMON <christian@momon.org>
|
|
*
|
|
* This file is part of StatoolInfos, simple service statistics tool.
|
|
*
|
|
* StatoolInfos is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as
|
|
* published by the Free Software Foundation, either version 3 of the
|
|
* License, or (at your option) any later version.
|
|
*
|
|
* StatoolInfos is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU Affero General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License
|
|
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
package fr.devinsy.statoolinfos.crawl;
|
|
|
|
import java.io.File;
|
|
import java.io.IOException;
|
|
import java.net.URL;
|
|
|
|
import org.apache.commons.codec.digest.DigestUtils;
|
|
import org.apache.commons.io.FileUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
import fr.devinsy.catgenerator.core.CatGenerator;
|
|
import fr.devinsy.statoolinfos.core.StatoolInfosException;
|
|
import fr.devinsy.statoolinfos.core.StatoolInfosUtils;
|
|
import fr.devinsy.statoolinfos.properties.PathProperties;
|
|
import fr.devinsy.statoolinfos.properties.PathPropertyList;
|
|
import fr.devinsy.statoolinfos.properties.PathPropertyUtils;
|
|
|
|
/**
|
|
* The Class CrawlCache.
|
|
*/
|
|
public class CrawlCache
|
|
{
|
|
private static Logger logger = LoggerFactory.getLogger(CrawlCache.class);
|
|
|
|
private File directory;
|
|
|
|
/**
|
|
* Instantiates a new crawl cache.
|
|
*
|
|
* @param directory
|
|
* the directory
|
|
* @throws StatoolInfosException
|
|
* the statool infos exception
|
|
*/
|
|
public CrawlCache(final File directory) throws StatoolInfosException
|
|
{
|
|
if (directory == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter.");
|
|
}
|
|
else if (StringUtils.isBlank(directory.getName()))
|
|
{
|
|
throw new IllegalArgumentException("Undefined directory.");
|
|
}
|
|
else if (!directory.exists())
|
|
{
|
|
throw new IllegalArgumentException("Directory does not exist.");
|
|
}
|
|
else
|
|
{
|
|
this.directory = directory;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Builds the file.
|
|
*
|
|
* @param key
|
|
* the key
|
|
* @return the file
|
|
*/
|
|
private File buildFile(final String key)
|
|
{
|
|
File result;
|
|
|
|
result = new File(this.directory, DigestUtils.md5Hex(key));
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Clear file in cache. Only files ending with ".property" are deleted.
|
|
*/
|
|
public void clear()
|
|
{
|
|
for (File file : this.directory.listFiles())
|
|
{
|
|
if ((file.isFile()) && ((file.getName().endsWith(".properties")) || file.getName().length() == 32))
|
|
{
|
|
logger.info("Deleting " + file.getName());
|
|
file.delete();
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Restore file.
|
|
*
|
|
* @param url
|
|
* the url
|
|
* @return the file
|
|
*/
|
|
public File restoreFile(final URL url)
|
|
{
|
|
File result;
|
|
|
|
if (url == null)
|
|
{
|
|
throw new IllegalArgumentException("Null parameter.");
|
|
}
|
|
else
|
|
{
|
|
result = buildFile(url.toString());
|
|
if (!result.exists())
|
|
{
|
|
result = null;
|
|
}
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Restore file to.
|
|
*
|
|
* @param url
|
|
* the url
|
|
* @param target
|
|
* the target
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public void restoreFileTo(final URL url, final File target) throws IOException
|
|
{
|
|
if (url != null)
|
|
{
|
|
File logoFile = restoreFile(url);
|
|
if (logoFile != null)
|
|
{
|
|
FileUtils.copyFile(logoFile, target);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Restore logo to.
|
|
*
|
|
* @param url
|
|
* the url
|
|
* @param target
|
|
* the target
|
|
* @param seed
|
|
* the seed
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public void restoreLogoTo(final URL url, final File target, final String seed) throws IOException
|
|
{
|
|
if ((target == null) || (seed == null))
|
|
{
|
|
throw new IllegalArgumentException("Null parameter.");
|
|
}
|
|
else
|
|
{
|
|
if (url == null)
|
|
{
|
|
try
|
|
{
|
|
logger.info("CatGeneratoring cat avatar (1): {}", target.getAbsoluteFile());
|
|
CatGenerator.buildAvatarTo(seed, target);
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
logger.warn("CatGeneratoring failed for {}: {}", seed, exception.getMessage());
|
|
StatoolInfosUtils.copyRessource("/fr/devinsy/statoolinfos/htmlize/stuff/default-organization-logo.png", target);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
File logoFile = restoreFile(url);
|
|
if (logoFile == null)
|
|
{
|
|
try
|
|
{
|
|
logger.info("CatGeneratoring cat avatar (2): {}", target.getAbsoluteFile());
|
|
CatGenerator.buildAvatarTo(seed, target);
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
logger.warn("CatGeneratoring failed for {}: {}", seed, exception.getMessage());
|
|
StatoolInfosUtils.copyRessource("/fr/devinsy/statoolinfos/htmlize/stuff/default-organization-logo.png", target);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
FileUtils.copyFile(logoFile, target);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Restore properties.
|
|
*
|
|
* @param url
|
|
* the url
|
|
* @return the path property list
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public PathProperties restoreProperties(final URL url) throws IOException
|
|
{
|
|
PathProperties result;
|
|
|
|
if (url == null)
|
|
{
|
|
result = new PathPropertyList();
|
|
}
|
|
else
|
|
{
|
|
File file = buildFile(url.toString() + ".properties");
|
|
result = PathPropertyUtils.load(file);
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Store.
|
|
*
|
|
* @param url
|
|
* the url
|
|
* @return the file
|
|
* @throws IOException
|
|
*/
|
|
public File store(final URL url) throws IOException
|
|
{
|
|
File result;
|
|
|
|
if (StringUtils.startsWith(url.getProtocol(), "http"))
|
|
{
|
|
final int TIMEOUT = 5000;
|
|
result = buildFile(url.toString());
|
|
FileUtils.copyURLToFile(url, result, TIMEOUT, TIMEOUT);
|
|
}
|
|
else
|
|
{
|
|
result = null;
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Store.
|
|
*
|
|
* @param url
|
|
* the url
|
|
* @param properties
|
|
* the properties
|
|
* @return the file
|
|
* @throws IOException
|
|
* Signals that an I/O exception has occurred.
|
|
*/
|
|
public File storeProperties(final URL url, final PathProperties properties) throws IOException
|
|
{
|
|
File result;
|
|
|
|
if ((url == null) || (!StringUtils.startsWith(url.getProtocol(), "http")))
|
|
{
|
|
result = null;
|
|
}
|
|
else
|
|
{
|
|
String key = url.toString() + ".properties";
|
|
result = buildFile(key);
|
|
|
|
PathPropertyUtils.save(result, properties);
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Store quietly.
|
|
*
|
|
* @param url
|
|
* the url
|
|
* @return the file
|
|
*/
|
|
public File storeQuietly(final URL url)
|
|
{
|
|
File result;
|
|
|
|
try
|
|
{
|
|
if ((url == null) || (!StringUtils.startsWith(url.getProtocol(), "http")))
|
|
{
|
|
result = null;
|
|
}
|
|
else
|
|
{
|
|
final int TIMEOUT = 5000;
|
|
result = buildFile(url.toString());
|
|
FileUtils.copyURLToFile(url, result, TIMEOUT, TIMEOUT);
|
|
logger.info("Crawled {}", url);
|
|
}
|
|
}
|
|
catch (IOException exception)
|
|
{
|
|
logger.info("Store faile for {}: {}", url, exception.getMessage());
|
|
result = null;
|
|
}
|
|
|
|
//
|
|
return result;
|
|
}
|
|
}
|