Compare commits
7 commits
214451155b
...
1178ab11a4
Author | SHA1 | Date | |
---|---|---|---|
1178ab11a4 | |||
ed886c6490 | |||
2f07f26d2a | |||
fb1407d332 | |||
5ce8d1ba78 | |||
96cbfb1062 | |||
c00b1ddcd8 |
50 changed files with 1312 additions and 78 deletions
10
.classpath
10
.classpath
|
@ -2,7 +2,7 @@
|
||||||
<classpath>
|
<classpath>
|
||||||
<classpathentry kind="src" path="src"/>
|
<classpathentry kind="src" path="src"/>
|
||||||
<classpathentry kind="src" path="test"/>
|
<classpathentry kind="src" path="test"/>
|
||||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11">
|
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-17">
|
||||||
<attributes>
|
<attributes>
|
||||||
<attribute name="module" value="true"/>
|
<attribute name="module" value="true"/>
|
||||||
<attribute name="owner.project.facets" value="java"/>
|
<attribute name="owner.project.facets" value="java"/>
|
||||||
|
@ -17,10 +17,10 @@
|
||||||
<classpathentry kind="lib" path="lib/UnitTesting/junit-4.12.jar" sourcepath="lib/UnitTesting/junit-4.12-sources.jar"/>
|
<classpathentry kind="lib" path="lib/UnitTesting/junit-4.12.jar" sourcepath="lib/UnitTesting/junit-4.12-sources.jar"/>
|
||||||
<classpathentry kind="lib" path="lib/commons-cli-1.4.jar" sourcepath="lib/commons-cli-1.4-sources.jar"/>
|
<classpathentry kind="lib" path="lib/commons-cli-1.4.jar" sourcepath="lib/commons-cli-1.4-sources.jar"/>
|
||||||
<classpathentry kind="lib" path="lib/commons-io-2.7.jar" sourcepath="lib/commons-io-2.7-sources.jar"/>
|
<classpathentry kind="lib" path="lib/commons-io-2.7.jar" sourcepath="lib/commons-io-2.7-sources.jar"/>
|
||||||
<classpathentry kind="lib" path="lib/commons-lang3-3.11.jar" sourcepath="lib/commons-lang3-3.11-sources.jar"/>
|
|
||||||
<classpathentry kind="lib" path="lib/commons-text-1.9.jar" sourcepath="lib/commons-text-1.9-sources.jar"/>
|
<classpathentry kind="lib" path="lib/commons-text-1.9.jar" sourcepath="lib/commons-text-1.9-sources.jar"/>
|
||||||
<classpathentry kind="lib" path="lib/devinsy-strings-0.11.3.jar" sourcepath="lib/devinsy-strings-0.11.3-sources.zip"/>
|
<classpathentry kind="lib" path="lib/commons-lang3-3.12.0.jar" sourcepath="lib/commons-lang3-3.12.0-sources.jar"/>
|
||||||
<classpathentry kind="lib" path="lib/threeten-extra-1.5.0.jar" sourcepath="lib/threeten-extra-1.5.0-sources.jar"/>
|
<classpathentry kind="lib" path="lib/devinsy-cmdexec-0.17.0.jar" sourcepath="lib/devinsy-cmdexec-0.17.0-sources.zip"/>
|
||||||
<classpathentry kind="lib" path="lib/devinsy-cmdexec-0.11.1.jar" sourcepath="lib/devinsy-cmdexec-0.11.1-sources.zip"/>
|
<classpathentry kind="lib" path="lib/devinsy-strings-0.17.0.jar" sourcepath="lib/devinsy-strings-0.17.0-sources.zip"/>
|
||||||
|
<classpathentry kind="lib" path="lib/threeten-extra-1.7.0.jar" sourcepath="lib/threeten-extra-1.7.0-sources.jar"/>
|
||||||
<classpathentry kind="output" path="bin"/>
|
<classpathentry kind="output" path="bin"/>
|
||||||
</classpath>
|
</classpath>
|
||||||
|
|
|
@ -11,9 +11,9 @@ org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes=
|
||||||
org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes=
|
org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes=
|
||||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
||||||
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
|
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
|
||||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=11
|
org.eclipse.jdt.core.compiler.codegen.targetPlatform=17
|
||||||
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
|
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
|
||||||
org.eclipse.jdt.core.compiler.compliance=11
|
org.eclipse.jdt.core.compiler.compliance=17
|
||||||
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
|
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
|
||||||
org.eclipse.jdt.core.compiler.debug.localVariable=generate
|
org.eclipse.jdt.core.compiler.debug.localVariable=generate
|
||||||
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
|
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
|
||||||
|
@ -22,7 +22,7 @@ org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
|
||||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
||||||
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
|
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning
|
||||||
org.eclipse.jdt.core.compiler.release=enabled
|
org.eclipse.jdt.core.compiler.release=enabled
|
||||||
org.eclipse.jdt.core.compiler.source=11
|
org.eclipse.jdt.core.compiler.source=17
|
||||||
org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns=false
|
org.eclipse.jdt.core.formatter.align_assignment_statements_on_columns=false
|
||||||
org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647
|
org.eclipse.jdt.core.formatter.align_fields_grouping_blank_lines=2147483647
|
||||||
org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
|
org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
|
||||||
|
|
|
@ -3,5 +3,5 @@
|
||||||
<fixed facet="java"/>
|
<fixed facet="java"/>
|
||||||
<fixed facet="jst.utility"/>
|
<fixed facet="jst.utility"/>
|
||||||
<installed facet="jst.utility" version="1.0"/>
|
<installed facet="jst.utility" version="1.0"/>
|
||||||
<installed facet="java" version="11"/>
|
<installed facet="java" version="17"/>
|
||||||
</faceted-project>
|
</faceted-project>
|
||||||
|
|
|
@ -111,7 +111,7 @@
|
||||||
<manifest>
|
<manifest>
|
||||||
<attribute name="Built-By" value="${user.name} using ant" />
|
<attribute name="Built-By" value="${user.name} using ant" />
|
||||||
<attribute name="Built-Date" value="${dist.time}" />
|
<attribute name="Built-Date" value="${dist.time}" />
|
||||||
<attribute name="Main-Class" value = "org.april.logar.LogarLauncher"/>
|
<attribute name="Main-Class" value = "fr.devinsy.logar.LogarLauncher"/>
|
||||||
</manifest>
|
</manifest>
|
||||||
<fileset dir="${build.classes}" />
|
<fileset dir="${build.classes}" />
|
||||||
<!--zipfileset dir="${basedir}/lib" includes="${toString:project.libs}" /-->
|
<!--zipfileset dir="${basedir}/lib" includes="${toString:project.libs}" /-->
|
||||||
|
|
|
@ -1,3 +1,3 @@
|
||||||
#Build Number for ANT. Do not edit!
|
#Build Number for ANT. Do not edit!
|
||||||
#Wed Apr 28 18:52:20 CEST 2021
|
#Wed Apr 28 18:52:20 CEST 2021
|
||||||
build.number=2
|
build.number=0
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
product.name=logar
|
product.name=logar
|
||||||
product.revision.major=0
|
product.revision.major=0
|
||||||
product.revision.minor=1
|
product.revision.minor=17
|
||||||
product.revision.snapshot=
|
product.revision.snapshot=
|
||||||
|
|
Binary file not shown.
Binary file not shown.
BIN
lib/commons-lang3-3.12.0-sources.jar
Normal file
BIN
lib/commons-lang3-3.12.0-sources.jar
Normal file
Binary file not shown.
BIN
lib/commons-lang3-3.12.0.jar
Normal file
BIN
lib/commons-lang3-3.12.0.jar
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
lib/threeten-extra-1.7.0-sources.jar
Normal file
BIN
lib/threeten-extra-1.7.0-sources.jar
Normal file
Binary file not shown.
BIN
lib/threeten-extra-1.7.0.jar
Normal file
BIN
lib/threeten-extra-1.7.0.jar
Normal file
Binary file not shown.
|
@ -3,7 +3,7 @@
|
||||||
|
|
||||||
# priority setting: DEBUG < INFO < WARN < ERROR
|
# priority setting: DEBUG < INFO < WARN < ERROR
|
||||||
log4j.rootLogger = INFO, console
|
log4j.rootLogger = INFO, console
|
||||||
log4j.logger.org.april.logar = INFO
|
log4j.logger.fr.devinsy.logar = INFO
|
||||||
|
|
||||||
#--
|
#--
|
||||||
log4j.appender.console = org.apache.log4j.ConsoleAppender
|
log4j.appender.console = org.apache.log4j.ConsoleAppender
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
|
|
||||||
# priority setting: DEBUG < INFO < WARN < ERROR
|
# priority setting: DEBUG < INFO < WARN < ERROR
|
||||||
log4j.rootLogger = INFO, stdout
|
log4j.rootLogger = INFO, stdout
|
||||||
log4j.logger.org.april.logar = INFO
|
log4j.logger.fr.devinsy.logar = INFO
|
||||||
|
|
||||||
#--
|
#--
|
||||||
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
|
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
|
||||||
|
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar;
|
package fr.devinsy.logar;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
|
||||||
|
@ -25,10 +25,11 @@ import org.apache.log4j.ConsoleAppender;
|
||||||
import org.apache.log4j.EnhancedPatternLayout;
|
import org.apache.log4j.EnhancedPatternLayout;
|
||||||
import org.apache.log4j.Level;
|
import org.apache.log4j.Level;
|
||||||
import org.apache.log4j.PropertyConfigurator;
|
import org.apache.log4j.PropertyConfigurator;
|
||||||
import org.april.logar.cli.LogarCLI;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import fr.devinsy.logar.cli.LogarCLI;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class LogarLauncher.
|
* The Class LogarLauncher.
|
||||||
*/
|
*/
|
||||||
|
@ -62,7 +63,7 @@ public final class LogarLauncher
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
BasicConfigurator.configure(new ConsoleAppender(new EnhancedPatternLayout("%m%n")));
|
BasicConfigurator.configure(new ConsoleAppender(new EnhancedPatternLayout("%m%n")));
|
||||||
org.apache.log4j.Logger.getRootLogger().setLevel(Level.INFO);
|
org.apache.log4j.Logger.getRootLogger().setLevel(Level.OFF);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run.
|
// Run.
|
|
@ -29,6 +29,13 @@ public enum ExtractOption
|
||||||
DATETIME,
|
DATETIME,
|
||||||
USERAGENT;
|
USERAGENT;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Of.
|
||||||
|
*
|
||||||
|
* @param source
|
||||||
|
* the source
|
||||||
|
* @return the extract option
|
||||||
|
*/
|
||||||
public static ExtractOption of(final String source)
|
public static ExtractOption of(final String source)
|
||||||
{
|
{
|
||||||
ExtractOption result;
|
ExtractOption result;
|
||||||
|
|
120
src/fr/devinsy/logar/app/ExtractOptions.java
Normal file
120
src/fr/devinsy/logar/app/ExtractOptions.java
Normal file
|
@ -0,0 +1,120 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.app;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Enum DryOption.
|
||||||
|
*/
|
||||||
|
public class ExtractOptions
|
||||||
|
{
|
||||||
|
OnOffOption ip;
|
||||||
|
OnOffOption user;
|
||||||
|
OnOffOption datetime;
|
||||||
|
OnOffOption userAgent;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new extract options.
|
||||||
|
*/
|
||||||
|
public ExtractOptions()
|
||||||
|
{
|
||||||
|
this.ip = OnOffOption.OFF;
|
||||||
|
this.user = OnOffOption.OFF;
|
||||||
|
this.datetime = OnOffOption.OFF;
|
||||||
|
this.userAgent = OnOffOption.OFF;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnOffOption getDatetime()
|
||||||
|
{
|
||||||
|
return this.datetime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnOffOption getIp()
|
||||||
|
{
|
||||||
|
return this.ip;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnOffOption getUser()
|
||||||
|
{
|
||||||
|
return this.user;
|
||||||
|
}
|
||||||
|
|
||||||
|
public OnOffOption getUserAgent()
|
||||||
|
{
|
||||||
|
return this.userAgent;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDatetime(final OnOffOption datetime)
|
||||||
|
{
|
||||||
|
this.datetime = datetime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIp(final OnOffOption ip)
|
||||||
|
{
|
||||||
|
this.ip = ip;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUser(final OnOffOption user)
|
||||||
|
{
|
||||||
|
this.user = user;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUserAgent(final OnOffOption userAgent)
|
||||||
|
{
|
||||||
|
this.userAgent = userAgent;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Of.
|
||||||
|
*
|
||||||
|
* @param source
|
||||||
|
* the source
|
||||||
|
* @return the extract option
|
||||||
|
*/
|
||||||
|
public static ExtractOptions of(final String source)
|
||||||
|
{
|
||||||
|
ExtractOptions result;
|
||||||
|
|
||||||
|
result = new ExtractOptions();
|
||||||
|
|
||||||
|
if (StringUtils.contains(source, "ip"))
|
||||||
|
{
|
||||||
|
result.setIp(OnOffOption.ON);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.contains(source, "remoteuser"))
|
||||||
|
{
|
||||||
|
result.setUser(OnOffOption.ON);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.contains(source, "datetime"))
|
||||||
|
{
|
||||||
|
result.setDatetime(OnOffOption.ON);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StringUtils.contains(source, "useragent"))
|
||||||
|
{
|
||||||
|
result.setUserAgent(OnOffOption.ON);
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
|
@ -29,9 +29,6 @@ import java.util.zip.GZIPOutputStream;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.april.logar.util.Files;
|
|
||||||
import org.april.logar.util.FilesUtils;
|
|
||||||
import org.april.logar.util.LineIterator;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@ -39,6 +36,12 @@ import fr.devinsy.logar.app.anonymizer.Anonymizer;
|
||||||
import fr.devinsy.logar.app.log.Log;
|
import fr.devinsy.logar.app.log.Log;
|
||||||
import fr.devinsy.logar.app.log.LogFile;
|
import fr.devinsy.logar.app.log.LogFile;
|
||||||
import fr.devinsy.logar.app.log.LogParser;
|
import fr.devinsy.logar.app.log.LogParser;
|
||||||
|
import fr.devinsy.logar.stats.UserAgentStator;
|
||||||
|
import fr.devinsy.logar.util.Chrono;
|
||||||
|
import fr.devinsy.logar.util.Files;
|
||||||
|
import fr.devinsy.logar.util.FilesUtils;
|
||||||
|
import fr.devinsy.logar.util.LineIterator;
|
||||||
|
import fr.devinsy.strings.StringList;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class Logar.
|
* The Class Logar.
|
||||||
|
@ -115,7 +118,7 @@ public final class Logar
|
||||||
* the target
|
* the target
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static void archive(final File source, final File target, final DryOption dry) throws IOException
|
public static void archive(final File source, final File target, final OnOffOption dry) throws IOException
|
||||||
{
|
{
|
||||||
archiveAccessFiles(source, target, dry);
|
archiveAccessFiles(source, target, dry);
|
||||||
archiveErrorFiles(source, target, dry);
|
archiveErrorFiles(source, target, dry);
|
||||||
|
@ -128,7 +131,7 @@ public final class Logar
|
||||||
* the source
|
* the source
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static void archiveAccessFiles(final File source, final File target, final DryOption dry) throws IOException
|
public static void archiveAccessFiles(final File source, final File target, final OnOffOption dry) throws IOException
|
||||||
{
|
{
|
||||||
if (source == null)
|
if (source == null)
|
||||||
{
|
{
|
||||||
|
@ -228,7 +231,7 @@ public final class Logar
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* Signals that an I/O exception has occurred.
|
* Signals that an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public static void archiveErrorFiles(final File source, final File target, final DryOption dry) throws IOException
|
public static void archiveErrorFiles(final File source, final File target, final OnOffOption dry) throws IOException
|
||||||
{
|
{
|
||||||
if (source == null)
|
if (source == null)
|
||||||
{
|
{
|
||||||
|
@ -503,13 +506,13 @@ public final class Logar
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* Signals that an I/O exception has occurred.
|
* Signals that an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public static void extract(final File source, final ExtractOption option) throws IOException
|
public static void extract(final File source, final ExtractOptions options) throws IOException
|
||||||
{
|
{
|
||||||
if (source == null)
|
if (source == null)
|
||||||
{
|
{
|
||||||
System.out.println("Undefined source.");
|
System.out.println("Undefined source.");
|
||||||
}
|
}
|
||||||
else if (option == null)
|
else if (options == null)
|
||||||
{
|
{
|
||||||
System.out.println("Undefined option.");
|
System.out.println("Undefined option.");
|
||||||
}
|
}
|
||||||
|
@ -535,26 +538,29 @@ public final class Logar
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
Log log = LogParser.parseAccessLog(line);
|
Log log = LogParser.parseAccessLog(line);
|
||||||
String extract;
|
|
||||||
switch (option)
|
StringList extract = new StringList();
|
||||||
|
if (options.getIp().isOn())
|
||||||
{
|
{
|
||||||
case IP:
|
extract.append(log.getIp());
|
||||||
extract = log.getIp();
|
|
||||||
break;
|
|
||||||
|
|
||||||
case DATETIME:
|
|
||||||
extract = log.getDatetimeValue();
|
|
||||||
break;
|
|
||||||
|
|
||||||
case USERAGENT:
|
|
||||||
extract = log.getUserAgent();
|
|
||||||
break;
|
|
||||||
|
|
||||||
default:
|
|
||||||
extract = null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
System.out.println(extract);
|
if (options.getUser().isOn())
|
||||||
|
{
|
||||||
|
extract.append(log.getUser());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.getDatetime().isOn())
|
||||||
|
{
|
||||||
|
extract.append(log.getDatetime());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.getUserAgent().isOn())
|
||||||
|
{
|
||||||
|
extract.append(log.getUserAgent());
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.println(extract.toStringSeparatedBy(" "));
|
||||||
}
|
}
|
||||||
catch (IllegalArgumentException exception)
|
catch (IllegalArgumentException exception)
|
||||||
{
|
{
|
||||||
|
@ -606,6 +612,113 @@ public final class Logar
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stats user agents.
|
||||||
|
*
|
||||||
|
* @param source
|
||||||
|
* the source
|
||||||
|
*/
|
||||||
|
public static void statUserAgents(final File source)
|
||||||
|
{
|
||||||
|
Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().sortByName();
|
||||||
|
|
||||||
|
for (File file : files)
|
||||||
|
{
|
||||||
|
statUserAgentsForFile(file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stat user agents for file.
|
||||||
|
*
|
||||||
|
* @param file
|
||||||
|
* the file
|
||||||
|
*/
|
||||||
|
public static void statUserAgentsForFile(final File file)
|
||||||
|
{
|
||||||
|
if (file == null)
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException("Null parameter.");
|
||||||
|
}
|
||||||
|
else if (!file.isFile())
|
||||||
|
{
|
||||||
|
throw new IllegalArgumentException("Parameter is not a file.");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
System.out.println("== Stat UserAgents for [" + file.getName() + "]");
|
||||||
|
|
||||||
|
int lineCount = 0;
|
||||||
|
int badLineCount = 0;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
UserAgentStator stator = new UserAgentStator();
|
||||||
|
LineIterator iterator = new LineIterator(file);
|
||||||
|
Chrono chrono = new Chrono().start();
|
||||||
|
long lastDuration = 0;
|
||||||
|
while (iterator.hasNext())
|
||||||
|
{
|
||||||
|
String line = iterator.next();
|
||||||
|
lineCount += 1;
|
||||||
|
|
||||||
|
if ((chrono.duration() % 60 == 0) && (chrono.duration() != lastDuration))
|
||||||
|
{
|
||||||
|
lastDuration = chrono.duration();
|
||||||
|
System.out.println(chrono.format() + " line counter " + lineCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Log log = LogParser.parseAccessLog(line);
|
||||||
|
stator.putLog(log);
|
||||||
|
}
|
||||||
|
catch (IllegalArgumentException exception)
|
||||||
|
{
|
||||||
|
System.out.println("Bad format line: " + line);
|
||||||
|
badLineCount += 1;
|
||||||
|
exception.printStackTrace();
|
||||||
|
}
|
||||||
|
catch (DateTimeParseException exception)
|
||||||
|
{
|
||||||
|
System.out.println("Bad datetime format: " + line);
|
||||||
|
badLineCount += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
System.out.println("Log count =" + stator.getLogCount());
|
||||||
|
System.out.println("Ip count =" + stator.getIps().size());
|
||||||
|
System.out.println("UserAgent count =" + stator.getUserAgents().size());
|
||||||
|
System.out.println("IpUserAgent count =" + stator.getIpUserAgents().size());
|
||||||
|
|
||||||
|
System.out.println("out=" + file.getCanonicalFile().getParentFile());
|
||||||
|
|
||||||
|
stator.saveIpList(new File(file.getParentFile(), "stator-ip.list"));
|
||||||
|
stator.saveUserAgentList(new File(file.getParentFile(), "stator-ua.list"));
|
||||||
|
stator.saveIpUserAgentList(new File(file.getParentFile(), "stator-ipua.list"));
|
||||||
|
|
||||||
|
stator.shrink();
|
||||||
|
System.out.println("UserAgent count =" + stator.getUserAgents().size());
|
||||||
|
System.out.println("IpUserAgent count =" + stator.getIpUserAgents().size());
|
||||||
|
stator.saveUserAgentList(new File(file.getParentFile(), "stator-ua-short.list"));
|
||||||
|
stator.saveIpUserAgentList(new File(file.getParentFile(), "stator-ipua-short.list"));
|
||||||
|
|
||||||
|
stator.computeIpLinkCountForUserAgent();
|
||||||
|
stator.saveUserAgentLinkCount(new File(file.getParentFile(), "stator-stats.list"));
|
||||||
|
}
|
||||||
|
catch (IOException exception)
|
||||||
|
{
|
||||||
|
System.err.println("Error with file [" + file.getAbsolutePath() + "]");
|
||||||
|
exception.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (badLineCount > 0)
|
||||||
|
{
|
||||||
|
System.out.println("Bad line count: " + badLineCount + "/" + lineCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check concate.
|
* Check concate.
|
||||||
*
|
*
|
||||||
|
|
|
@ -21,7 +21,7 @@ package fr.devinsy.logar.app;
|
||||||
/**
|
/**
|
||||||
* The Enum DryOption.
|
* The Enum DryOption.
|
||||||
*/
|
*/
|
||||||
public enum DryOption
|
public enum OnOffOption
|
||||||
{
|
{
|
||||||
ON,
|
ON,
|
||||||
OFF;
|
OFF;
|
|
@ -29,12 +29,12 @@ import java.util.zip.GZIPOutputStream;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
import org.april.logar.util.LineIterator;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import fr.devinsy.logar.app.log.Log;
|
import fr.devinsy.logar.app.log.Log;
|
||||||
import fr.devinsy.logar.app.log.LogParser;
|
import fr.devinsy.logar.app.log.LogParser;
|
||||||
|
import fr.devinsy.logar.util.LineIterator;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class Anonymizer.
|
* The Class Anonymizer.
|
||||||
|
@ -119,6 +119,16 @@ public final class Anonymizer
|
||||||
// anon.getUser(), anon.getDatetime());
|
// anon.getUser(), anon.getDatetime());
|
||||||
// logger.info("anon={}", anon);
|
// logger.info("anon={}", anon);
|
||||||
|
|
||||||
|
// System.out.println("\\\\\\" + line);
|
||||||
|
// System.out.println("///" + anon.getLine());
|
||||||
|
|
||||||
|
// String[] tokens = log.getUserAgent().split("[
|
||||||
|
// ()\\[\\]/]");
|
||||||
|
// for (String token : tokens)
|
||||||
|
// {
|
||||||
|
// out.println(token);
|
||||||
|
// }
|
||||||
|
|
||||||
// Make a check.
|
// Make a check.
|
||||||
if (StringUtils.equals(line, anon.getLine()))
|
if (StringUtils.equals(line, anon.getLine()))
|
||||||
{
|
{
|
||||||
|
@ -158,7 +168,7 @@ public final class Anonymizer
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Anonymize.
|
* Anonymize access.
|
||||||
*
|
*
|
||||||
* @param log
|
* @param log
|
||||||
* the log
|
* the log
|
||||||
|
@ -168,6 +178,42 @@ public final class Anonymizer
|
||||||
{
|
{
|
||||||
Log result;
|
Log result;
|
||||||
|
|
||||||
|
result = new Log(log);
|
||||||
|
|
||||||
|
//
|
||||||
|
String anonIp = this.map.anonymizeIp(log.getIp());
|
||||||
|
result.setIp(anonIp);
|
||||||
|
|
||||||
|
//
|
||||||
|
String refererAnon = log.getReferer().replace(log.getIp(), result.getIp());
|
||||||
|
result.setReferer(refererAnon);
|
||||||
|
|
||||||
|
//
|
||||||
|
String anonUser = this.map.anonymizeUser(log.getUser());
|
||||||
|
result.setUser(anonUser);
|
||||||
|
|
||||||
|
//
|
||||||
|
String userAgentAnon = UserAgentGenerator.anonymize(log.getUserAgent());
|
||||||
|
result.setUserAgent(userAgentAnon);
|
||||||
|
|
||||||
|
//
|
||||||
|
result.concateAccessLog();
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Anonymize.
|
||||||
|
*
|
||||||
|
* @param log
|
||||||
|
* the log
|
||||||
|
* @return the log
|
||||||
|
*/
|
||||||
|
public Log anonymizeAccessSimple(final Log log)
|
||||||
|
{
|
||||||
|
Log result;
|
||||||
|
|
||||||
//
|
//
|
||||||
String anonIp = this.map.anonymizeIp(log.getIp());
|
String anonIp = this.map.anonymizeIp(log.getIp());
|
||||||
String line = log.getLine().replace(log.getIp(), anonIp);
|
String line = log.getLine().replace(log.getIp(), anonIp);
|
||||||
|
|
|
@ -24,14 +24,14 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class RandomIpv4Generator.
|
* The Class Ipv4Generator.
|
||||||
*/
|
*/
|
||||||
public final class Ipv4Generator
|
public final class Ipv4Generator
|
||||||
{
|
{
|
||||||
private static Logger logger = LoggerFactory.getLogger(Ipv4Generator.class);
|
private static Logger logger = LoggerFactory.getLogger(Ipv4Generator.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Instantiates a new random ipv 4 generator.
|
* Instantiates a new ipv 4 generator.
|
||||||
*/
|
*/
|
||||||
private Ipv4Generator()
|
private Ipv4Generator()
|
||||||
{
|
{
|
||||||
|
|
|
@ -24,14 +24,14 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class RandomIpv6Generator.
|
* The Class Ipv6Generator.
|
||||||
*/
|
*/
|
||||||
public final class Ipv6Generator
|
public final class Ipv6Generator
|
||||||
{
|
{
|
||||||
private static Logger logger = LoggerFactory.getLogger(Ipv6Generator.class);
|
private static Logger logger = LoggerFactory.getLogger(Ipv6Generator.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Instantiates a new random ipv 6 generator.
|
* Instantiates a new ipv 6 generator.
|
||||||
*/
|
*/
|
||||||
private Ipv6Generator()
|
private Ipv6Generator()
|
||||||
{
|
{
|
||||||
|
|
87
src/fr/devinsy/logar/app/anonymizer/UserAgentGenerator.java
Normal file
87
src/fr/devinsy/logar/app/anonymizer/UserAgentGenerator.java
Normal file
|
@ -0,0 +1,87 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.app.anonymizer;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.RegExUtils;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class UserAgentGenerator.
|
||||||
|
*/
|
||||||
|
public final class UserAgentGenerator
|
||||||
|
{
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(UserAgentGenerator.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new user agent generator.
|
||||||
|
*/
|
||||||
|
private UserAgentGenerator()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shrink.
|
||||||
|
*
|
||||||
|
* @param source
|
||||||
|
* the source
|
||||||
|
* @return the string
|
||||||
|
*/
|
||||||
|
public static String anonymize(final String source)
|
||||||
|
{
|
||||||
|
String result;
|
||||||
|
|
||||||
|
if (source == null)
|
||||||
|
{
|
||||||
|
result = null;
|
||||||
|
}
|
||||||
|
else if (source.equals("-"))
|
||||||
|
{
|
||||||
|
result = source;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// StringBuffer buffer = new StringBuffer(source.length());
|
||||||
|
// String targets = "./[]_-0123456789";
|
||||||
|
// for (int index = 0; index < source.length(); index++)
|
||||||
|
// {
|
||||||
|
// char letter = source.charAt(index);
|
||||||
|
// if (!StringUtils.containsAny(targets, letter))
|
||||||
|
// {
|
||||||
|
// buffer.append(letter);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
result = source;
|
||||||
|
result = RegExUtils.replaceAll(result, "\\[.+\\]", "");
|
||||||
|
result = RegExUtils.replaceAll(result, "https?://[^\\s\\)\\],;]+", "");
|
||||||
|
result = RegExUtils.replaceAll(result, "\\(\\)", "");
|
||||||
|
result = RegExUtils.replaceAll(result, "(B|b)uild ?[~0-9a-zA-Z/\\.-_@]+", "build");
|
||||||
|
result = RegExUtils.replaceAll(result, "\\d{4}/\\d{2}/\\d{2}", "");
|
||||||
|
result = RegExUtils.replaceAll(result, "(/|:|-|@)?[0-9][~0-9a-zA-Z\\.-_]+", "");
|
||||||
|
// result = StringUtils.replaceChars(result, ".[]_-0123456789", "");
|
||||||
|
|
||||||
|
result = StringUtils.rightPad(result, source.length());
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
|
@ -23,14 +23,14 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class RandomUserGenerator.
|
* The Class UserGenerator.
|
||||||
*/
|
*/
|
||||||
public final class UserGenerator
|
public final class UserGenerator
|
||||||
{
|
{
|
||||||
private static Logger logger = LoggerFactory.getLogger(UserGenerator.class);
|
private static Logger logger = LoggerFactory.getLogger(UserGenerator.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Instantiates a new random user generator.
|
* Instantiates a new user generator.
|
||||||
*/
|
*/
|
||||||
private UserGenerator()
|
private UserGenerator()
|
||||||
{
|
{
|
||||||
|
|
|
@ -21,7 +21,7 @@ package fr.devinsy.logar.app.log;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
|
|
||||||
import org.april.logar.util.CompareUtils;
|
import fr.devinsy.logar.util.CompareUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class LogComparator.
|
* The Class LogComparator.
|
||||||
|
|
|
@ -26,12 +26,12 @@ import java.io.PrintWriter;
|
||||||
import java.util.zip.GZIPOutputStream;
|
import java.util.zip.GZIPOutputStream;
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.april.logar.util.LineIterator;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import fr.devinsy.cmdexec.CmdExecException;
|
import fr.devinsy.cmdexec.CmdExecException;
|
||||||
import fr.devinsy.cmdexec.CmdExecUtils;
|
import fr.devinsy.cmdexec.CmdExecUtils;
|
||||||
|
import fr.devinsy.logar.util.LineIterator;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Class LogFile.
|
* The Class LogFile.
|
||||||
|
|
|
@ -16,18 +16,18 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.cli;
|
package fr.devinsy.logar.cli;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
|
|
||||||
import org.april.logar.util.BuildInformation;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import fr.devinsy.logar.app.DryOption;
|
import fr.devinsy.logar.app.ExtractOptions;
|
||||||
import fr.devinsy.logar.app.ExtractOption;
|
|
||||||
import fr.devinsy.logar.app.Logar;
|
import fr.devinsy.logar.app.Logar;
|
||||||
|
import fr.devinsy.logar.app.OnOffOption;
|
||||||
|
import fr.devinsy.logar.util.BuildInformation;
|
||||||
import fr.devinsy.strings.StringList;
|
import fr.devinsy.strings.StringList;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -56,14 +56,15 @@ public final class LogarCLI
|
||||||
message.appendln("Usage:");
|
message.appendln("Usage:");
|
||||||
message.appendln(" logar [ -h | -help | --help ]");
|
message.appendln(" logar [ -h | -help | --help ]");
|
||||||
message.appendln(" logar [ -v | -version | --version ]");
|
message.appendln(" logar [ -v | -version | --version ]");
|
||||||
message.appendln(" logar anonymize fileordirectory [mapfile] anonymize ip and user");
|
message.appendln(" logar anonymize <fileordirectory> [<mapfile>] anonymize ip and user");
|
||||||
message.appendln(" logar archive [-dry] source target archive previous month from /var/log/nginx/ tree");
|
message.appendln(" logar archive [-dry] <source> <target> archive previous month from /var/log/nginx/ tree");
|
||||||
message.appendln(" logar check fileordirectory check line format in log file");
|
message.appendln(" logar check <fileordirectory> check line format in log files");
|
||||||
message.appendln(" logar checksort fileordirectory check sort in log file");
|
message.appendln(" logar checksort <fileordirectory> check sort in log files");
|
||||||
message.appendln(" logar sort fileordirectory sort log files by datetime");
|
message.appendln(" logar extract <fields> <fileordirectory> extract one or more fields (ip,user,datetime,useragent) from log files");
|
||||||
message.appendln(" logar testconcate fileordirectory test line concate in log file");
|
message.appendln(" logar sort <fileordirectory> sort log files by datetime");
|
||||||
|
message.appendln(" logar testconcate <fileordirectory> test line concate in log files");
|
||||||
|
|
||||||
logger.info(message.toString());
|
System.out.println(message.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -75,7 +76,7 @@ public final class LogarCLI
|
||||||
|
|
||||||
message.appendln(BuildInformation.instance().version());
|
message.appendln(BuildInformation.instance().version());
|
||||||
|
|
||||||
logger.info(message.toString());
|
System.out.println(message.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -203,14 +204,14 @@ public final class LogarCLI
|
||||||
File source = new File(args[1]);
|
File source = new File(args[1]);
|
||||||
File target = new File(args[2]);
|
File target = new File(args[2]);
|
||||||
|
|
||||||
Logar.archive(source, target, DryOption.OFF);
|
Logar.archive(source, target, OnOffOption.OFF);
|
||||||
}
|
}
|
||||||
else if (isMatching(args, "archive", "-dry", "\\s*\\S+\\s*", "\\s*\\S+\\s*"))
|
else if (isMatching(args, "archive", "-dry", "\\s*\\S+\\s*", "\\s*\\S+\\s*"))
|
||||||
{
|
{
|
||||||
File source = new File(args[2]);
|
File source = new File(args[2]);
|
||||||
File target = new File(args[3]);
|
File target = new File(args[3]);
|
||||||
|
|
||||||
Logar.archive(source, target, DryOption.ON);
|
Logar.archive(source, target, OnOffOption.ON);
|
||||||
}
|
}
|
||||||
else if (isMatching(args, "check", "\\s*\\S+\\s*"))
|
else if (isMatching(args, "check", "\\s*\\S+\\s*"))
|
||||||
{
|
{
|
||||||
|
@ -224,12 +225,12 @@ public final class LogarCLI
|
||||||
|
|
||||||
Logar.checkSort(source);
|
Logar.checkSort(source);
|
||||||
}
|
}
|
||||||
else if (isMatching(args, "extract", "\\s*(ip|datetime|useragent)\\s*", "\\s*\\S+\\s*"))
|
else if (isMatching(args, "extract", "\\s*((ip)?(,)?(remoteuser)?(,)?(datetime)?(,)?(useragent)?)\\s*", "\\s*\\S+\\s*"))
|
||||||
{
|
{
|
||||||
ExtractOption token = ExtractOption.of(args[1]);
|
ExtractOptions options = ExtractOptions.of(args[1]);
|
||||||
File source = new File(args[2]);
|
File source = new File(args[2]);
|
||||||
|
|
||||||
Logar.extract(source, token);
|
Logar.extract(source, options);
|
||||||
}
|
}
|
||||||
else if (isMatching(args, "sort", "\\s*\\S+\\s*"))
|
else if (isMatching(args, "sort", "\\s*\\S+\\s*"))
|
||||||
{
|
{
|
||||||
|
@ -237,6 +238,12 @@ public final class LogarCLI
|
||||||
|
|
||||||
Logar.sort(source);
|
Logar.sort(source);
|
||||||
}
|
}
|
||||||
|
else if (isMatching(args, "statuseragent", "\\s*\\S+\\s*"))
|
||||||
|
{
|
||||||
|
File source = new File(args[1]);
|
||||||
|
|
||||||
|
Logar.statUserAgents(source);
|
||||||
|
}
|
||||||
else if (isMatching(args, "testconcate", "\\s*\\S+\\s*"))
|
else if (isMatching(args, "testconcate", "\\s*\\S+\\s*"))
|
||||||
{
|
{
|
||||||
File source = new File(args[1]);
|
File source = new File(args[1]);
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.cli;
|
package fr.devinsy.logar.cli;
|
||||||
|
|
||||||
import fr.devinsy.logar.app.LogarException;
|
import fr.devinsy.logar.app.LogarException;
|
||||||
|
|
60
src/fr/devinsy/logar/stats/Ip.java
Normal file
60
src/fr/devinsy/logar/stats/Ip.java
Normal file
|
@ -0,0 +1,60 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.stats;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class IpStat.
|
||||||
|
*/
|
||||||
|
public final class Ip
|
||||||
|
{
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(Ip.class);
|
||||||
|
|
||||||
|
private String value;
|
||||||
|
private long count;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new ip stat.
|
||||||
|
*
|
||||||
|
* @param ip
|
||||||
|
* the ip
|
||||||
|
*/
|
||||||
|
public Ip(final String ip)
|
||||||
|
{
|
||||||
|
this.value = ip;
|
||||||
|
this.count = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getCount()
|
||||||
|
{
|
||||||
|
return this.count;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getValue()
|
||||||
|
{
|
||||||
|
return this.value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void inc()
|
||||||
|
{
|
||||||
|
this.count += 1;
|
||||||
|
}
|
||||||
|
}
|
70
src/fr/devinsy/logar/stats/IpUserAgent.java
Normal file
70
src/fr/devinsy/logar/stats/IpUserAgent.java
Normal file
|
@ -0,0 +1,70 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.stats;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class UserAgentStat.
|
||||||
|
*/
|
||||||
|
public final class IpUserAgent
|
||||||
|
{
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(IpUserAgent.class);
|
||||||
|
|
||||||
|
private String ip;
|
||||||
|
private String userAgent;
|
||||||
|
private long count;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new user agent stat.
|
||||||
|
*
|
||||||
|
* @param ip
|
||||||
|
* the ip
|
||||||
|
* @param userAgent
|
||||||
|
* the user agent
|
||||||
|
*/
|
||||||
|
public IpUserAgent(final String ip, final String userAgent)
|
||||||
|
{
|
||||||
|
this.ip = ip;
|
||||||
|
this.userAgent = userAgent;
|
||||||
|
this.count = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getCount()
|
||||||
|
{
|
||||||
|
return this.count;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getIp()
|
||||||
|
{
|
||||||
|
return this.ip;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUserAgent()
|
||||||
|
{
|
||||||
|
return this.userAgent;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void inc()
|
||||||
|
{
|
||||||
|
this.count += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
42
src/fr/devinsy/logar/stats/IpUserAgentList.java
Normal file
42
src/fr/devinsy/logar/stats/IpUserAgentList.java
Normal file
|
@ -0,0 +1,42 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.stats;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class IpUserAgentStatList.
|
||||||
|
*/
|
||||||
|
public final class IpUserAgentList extends ArrayList<IpUserAgent>
|
||||||
|
{
|
||||||
|
private static final long serialVersionUID = 385645188228694639L;
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(IpUserAgentList.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new ip user agent stats.
|
||||||
|
*/
|
||||||
|
public IpUserAgentList()
|
||||||
|
{
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
}
|
143
src/fr/devinsy/logar/stats/IpUserAgents.java
Normal file
143
src/fr/devinsy/logar/stats/IpUserAgents.java
Normal file
|
@ -0,0 +1,143 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.stats;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Iterator;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class IpUserAgentStats.
|
||||||
|
*/
|
||||||
|
public final class IpUserAgents implements Iterable<IpUserAgent>
|
||||||
|
{
|
||||||
|
private static final long serialVersionUID = -3011532898476944152L;
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(IpUserAgents.class);
|
||||||
|
|
||||||
|
private HashMap<String, IpUserAgent> data;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new ip user agent stats.
|
||||||
|
*/
|
||||||
|
public IpUserAgents()
|
||||||
|
{
|
||||||
|
this.data = new HashMap<String, IpUserAgent>();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void clear()
|
||||||
|
{
|
||||||
|
this.data.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
public int countByUserAgent(final String userAgent)
|
||||||
|
{
|
||||||
|
int result;
|
||||||
|
|
||||||
|
result = 0;
|
||||||
|
|
||||||
|
for (IpUserAgent ipUserAgent : this.data.values())
|
||||||
|
{
|
||||||
|
if (StringUtils.equals(ipUserAgent.getUserAgent(), userAgent))
|
||||||
|
{
|
||||||
|
result += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the by user agent.
|
||||||
|
*
|
||||||
|
* @param userAgent
|
||||||
|
* the user agent
|
||||||
|
* @return the by user agent
|
||||||
|
*/
|
||||||
|
public IpUserAgentList getByUserAgent(final String userAgent)
|
||||||
|
{
|
||||||
|
IpUserAgentList result;
|
||||||
|
|
||||||
|
result = new IpUserAgentList();
|
||||||
|
|
||||||
|
for (IpUserAgent ipUserAgent : this.data.values())
|
||||||
|
{
|
||||||
|
if (StringUtils.equals(ipUserAgent.getUserAgent(), userAgent))
|
||||||
|
{
|
||||||
|
result.add(ipUserAgent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* (non-Javadoc)
|
||||||
|
* @see java.lang.Iterable#iterator()
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public Iterator<IpUserAgent> iterator()
|
||||||
|
{
|
||||||
|
Iterator<IpUserAgent> result;
|
||||||
|
|
||||||
|
result = this.data.values().iterator();
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void put(final IpUserAgent ipUserAgent)
|
||||||
|
{
|
||||||
|
this.data.put(ipUserAgent.getIp() + ipUserAgent.getUserAgent(), ipUserAgent);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put.
|
||||||
|
*
|
||||||
|
* @param ip
|
||||||
|
* the ip
|
||||||
|
* @param userAgent
|
||||||
|
* the user agent
|
||||||
|
*/
|
||||||
|
public void put(final String ip, final String userAgent)
|
||||||
|
{
|
||||||
|
IpUserAgent stat = this.data.get(ip + userAgent);
|
||||||
|
if (stat == null)
|
||||||
|
{
|
||||||
|
stat = new IpUserAgent(ip, userAgent);
|
||||||
|
this.data.put(ip + userAgent, stat);
|
||||||
|
}
|
||||||
|
|
||||||
|
stat.inc();
|
||||||
|
}
|
||||||
|
|
||||||
|
public int size()
|
||||||
|
{
|
||||||
|
int result;
|
||||||
|
|
||||||
|
result = this.data.size();
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
54
src/fr/devinsy/logar/stats/Ips.java
Normal file
54
src/fr/devinsy/logar/stats/Ips.java
Normal file
|
@ -0,0 +1,54 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.stats;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class IpStats.
|
||||||
|
*/
|
||||||
|
public final class Ips extends HashMap<String, Ip>
|
||||||
|
{
|
||||||
|
private static final long serialVersionUID = -8411746796941831991L;
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(Ips.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new ip stats.
|
||||||
|
*/
|
||||||
|
public Ips()
|
||||||
|
{
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void put(final String ip)
|
||||||
|
{
|
||||||
|
Ip stat = get(ip);
|
||||||
|
if (stat == null)
|
||||||
|
{
|
||||||
|
stat = new Ip(ip);
|
||||||
|
this.put(ip, stat);
|
||||||
|
}
|
||||||
|
|
||||||
|
stat.inc();
|
||||||
|
}
|
||||||
|
}
|
75
src/fr/devinsy/logar/stats/UserAgent.java
Normal file
75
src/fr/devinsy/logar/stats/UserAgent.java
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.stats;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class UserAgentStat.
|
||||||
|
*/
|
||||||
|
public final class UserAgent
|
||||||
|
{
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(UserAgent.class);
|
||||||
|
|
||||||
|
private String value;
|
||||||
|
private long count;
|
||||||
|
private long ipLinkCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new user agent stat.
|
||||||
|
*
|
||||||
|
* @param ip
|
||||||
|
* the ip
|
||||||
|
* @param userAgent
|
||||||
|
* the user agent
|
||||||
|
*/
|
||||||
|
public UserAgent(final String userAgent)
|
||||||
|
{
|
||||||
|
this.value = userAgent;
|
||||||
|
this.count = 0;
|
||||||
|
this.ipLinkCount = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getCount()
|
||||||
|
{
|
||||||
|
return this.count;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getIpLinkCount()
|
||||||
|
{
|
||||||
|
return this.ipLinkCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getValue()
|
||||||
|
{
|
||||||
|
return this.value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void inc()
|
||||||
|
{
|
||||||
|
this.count += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIpLinkCount(final long ipLinkCount)
|
||||||
|
{
|
||||||
|
this.ipLinkCount = ipLinkCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
232
src/fr/devinsy/logar/stats/UserAgentStator.java
Normal file
232
src/fr/devinsy/logar/stats/UserAgentStator.java
Normal file
|
@ -0,0 +1,232 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.stats;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import fr.devinsy.logar.app.log.Log;
|
||||||
|
import fr.devinsy.logar.util.Chrono;
|
||||||
|
import fr.devinsy.strings.StringsUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class UserAgentStator.
|
||||||
|
*/
|
||||||
|
public final class UserAgentStator
|
||||||
|
{
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(UserAgentStator.class);
|
||||||
|
|
||||||
|
private long logCount;
|
||||||
|
private Ips ips;
|
||||||
|
private UserAgents userAgents;
|
||||||
|
private IpUserAgents ipUserAgents;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new user agent stator.
|
||||||
|
*/
|
||||||
|
public UserAgentStator()
|
||||||
|
{
|
||||||
|
this.logCount = 0;
|
||||||
|
this.ips = new Ips();
|
||||||
|
this.userAgents = new UserAgents();
|
||||||
|
this.ipUserAgents = new IpUserAgents();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute ip link count for user agent.
|
||||||
|
*/
|
||||||
|
public void computeIpLinkCountForUserAgent()
|
||||||
|
{
|
||||||
|
int index = 0;
|
||||||
|
Chrono chrono = new Chrono().start();
|
||||||
|
long lastDuration = 0;
|
||||||
|
for (UserAgent userAgent : this.userAgents.values())
|
||||||
|
{
|
||||||
|
index += 1;
|
||||||
|
|
||||||
|
if ((chrono.duration() % 60 == 0) && (chrono.duration() != lastDuration))
|
||||||
|
{
|
||||||
|
lastDuration = chrono.duration();
|
||||||
|
System.out.println(chrono.format() + " IpLinkCount " + index + "/" + this.userAgents.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
long count = this.ipUserAgents.countByUserAgent(userAgent.getValue());
|
||||||
|
|
||||||
|
userAgent.setIpLinkCount(count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Ips getIps()
|
||||||
|
{
|
||||||
|
return this.ips;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IpUserAgents getIpUserAgents()
|
||||||
|
{
|
||||||
|
return this.ipUserAgents;
|
||||||
|
}
|
||||||
|
|
||||||
|
public long getLogCount()
|
||||||
|
{
|
||||||
|
return this.logCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
public UserAgents getUserAgents()
|
||||||
|
{
|
||||||
|
return this.userAgents;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds the log.
|
||||||
|
*
|
||||||
|
* @param log
|
||||||
|
* the log
|
||||||
|
*/
|
||||||
|
public void putLog(final Log log)
|
||||||
|
{
|
||||||
|
String userAgent = log.getUserAgent().trim();
|
||||||
|
|
||||||
|
this.logCount += 1;
|
||||||
|
this.ips.put(log.getIp());
|
||||||
|
this.userAgents.put(userAgent);
|
||||||
|
this.ipUserAgents.put(log.getIp(), userAgent);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void saveIpList(final File target)
|
||||||
|
{
|
||||||
|
PrintWriter out = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
out = new PrintWriter(new FileOutputStream(target));
|
||||||
|
|
||||||
|
for (Ip ip : this.ips.values())
|
||||||
|
{
|
||||||
|
out.println(ip.getValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException exception)
|
||||||
|
{
|
||||||
|
exception.printStackTrace();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
IOUtils.closeQuietly(out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void saveIpUserAgentList(final File target)
|
||||||
|
{
|
||||||
|
PrintWriter out = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
out = new PrintWriter(new FileOutputStream(target));
|
||||||
|
|
||||||
|
for (IpUserAgent userAgent : this.ipUserAgents)
|
||||||
|
{
|
||||||
|
out.println(userAgent.getIp() + " " + userAgent.getUserAgent());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException exception)
|
||||||
|
{
|
||||||
|
exception.printStackTrace();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
IOUtils.closeQuietly(out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void saveUserAgentLinkCount(final File target)
|
||||||
|
{
|
||||||
|
PrintWriter out = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
out = new PrintWriter(new FileOutputStream(target));
|
||||||
|
for (UserAgent userAgent : this.userAgents.values())
|
||||||
|
{
|
||||||
|
out.println(userAgent.getIpLinkCount() + " " + userAgent.getCount() + " " + userAgent.getValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException exception)
|
||||||
|
{
|
||||||
|
exception.printStackTrace();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
IOUtils.closeQuietly(out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void saveUserAgentList(final File target)
|
||||||
|
{
|
||||||
|
PrintWriter out = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
out = new PrintWriter(new FileOutputStream(target));
|
||||||
|
|
||||||
|
for (UserAgent userAgent : this.userAgents.values())
|
||||||
|
{
|
||||||
|
out.println(userAgent.getValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (FileNotFoundException exception)
|
||||||
|
{
|
||||||
|
exception.printStackTrace();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
IOUtils.closeQuietly(out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void shrink()
|
||||||
|
{
|
||||||
|
String[] tokens = { "android", "apple", "chrome", "iphone", "linux", "mac", "mozilla", "opera", "safari", "thunderbird" };
|
||||||
|
|
||||||
|
//
|
||||||
|
UserAgents userAgentBis = new UserAgents();
|
||||||
|
for (UserAgent userAgent : this.userAgents.values())
|
||||||
|
{
|
||||||
|
if (StringsUtils.containsAnyIgnoreCase(userAgent.getValue(), tokens))
|
||||||
|
{
|
||||||
|
userAgentBis.put(userAgent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.userAgents.clear();
|
||||||
|
this.userAgents = userAgentBis;
|
||||||
|
|
||||||
|
//
|
||||||
|
IpUserAgents ipUserAgentBis = new IpUserAgents();
|
||||||
|
for (IpUserAgent ipUserAgent : this.ipUserAgents)
|
||||||
|
{
|
||||||
|
if (StringsUtils.containsAnyIgnoreCase(ipUserAgent.getUserAgent(), tokens))
|
||||||
|
{
|
||||||
|
ipUserAgentBis.put(ipUserAgent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.ipUserAgents.clear();
|
||||||
|
this.ipUserAgents = ipUserAgentBis;
|
||||||
|
}
|
||||||
|
}
|
59
src/fr/devinsy/logar/stats/UserAgents.java
Normal file
59
src/fr/devinsy/logar/stats/UserAgents.java
Normal file
|
@ -0,0 +1,59 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.stats;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class UserAgents.
|
||||||
|
*/
|
||||||
|
public final class UserAgents extends HashMap<String, UserAgent>
|
||||||
|
{
|
||||||
|
private static final long serialVersionUID = -7943808966632477322L;
|
||||||
|
|
||||||
|
private static Logger logger = LoggerFactory.getLogger(UserAgents.class);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new user agents.
|
||||||
|
*/
|
||||||
|
public UserAgents()
|
||||||
|
{
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void put(final String userAgent)
|
||||||
|
{
|
||||||
|
UserAgent stat = get(userAgent);
|
||||||
|
if (stat == null)
|
||||||
|
{
|
||||||
|
stat = new UserAgent(userAgent);
|
||||||
|
this.put(userAgent, stat);
|
||||||
|
}
|
||||||
|
|
||||||
|
stat.inc();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void put(final UserAgent userAgent)
|
||||||
|
{
|
||||||
|
this.put(userAgent.getValue(), userAgent);
|
||||||
|
}
|
||||||
|
}
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.util;
|
package fr.devinsy.logar.util;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
118
src/fr/devinsy/logar/util/Chrono.java
Normal file
118
src/fr/devinsy/logar/util/Chrono.java
Normal file
|
@ -0,0 +1,118 @@
|
||||||
|
/*
|
||||||
|
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
|
||||||
|
*
|
||||||
|
* This file is part of Logar, simple tool to manage http log files.
|
||||||
|
*
|
||||||
|
* Logar is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* Logar is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package fr.devinsy.logar.util;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The Class Chrono.
|
||||||
|
*/
|
||||||
|
public class Chrono
|
||||||
|
{
|
||||||
|
private LocalDateTime start;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Instantiates a new time keeper.
|
||||||
|
*/
|
||||||
|
public Chrono()
|
||||||
|
{
|
||||||
|
reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Duration.
|
||||||
|
*
|
||||||
|
* @return the long
|
||||||
|
*/
|
||||||
|
public long duration()
|
||||||
|
{
|
||||||
|
long result;
|
||||||
|
|
||||||
|
result = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC) - this.start.toEpochSecond(ZoneOffset.UTC);
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Format.
|
||||||
|
*
|
||||||
|
* @return the string
|
||||||
|
*/
|
||||||
|
public String format()
|
||||||
|
{
|
||||||
|
String result;
|
||||||
|
|
||||||
|
if (this.start == null)
|
||||||
|
{
|
||||||
|
result = "n/a";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
LocalDateTime end = LocalDateTime.now();
|
||||||
|
Duration duration = Duration.between(this.start, end);
|
||||||
|
|
||||||
|
result = String.format("%02d:%02d:%02d", duration.getSeconds() / 60 / 60, duration.getSeconds() / 60, duration.getSeconds() % 60);
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset.
|
||||||
|
*/
|
||||||
|
public void reset()
|
||||||
|
{
|
||||||
|
this.start = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start.
|
||||||
|
*/
|
||||||
|
public Chrono start()
|
||||||
|
{
|
||||||
|
Chrono result;
|
||||||
|
|
||||||
|
this.start = LocalDateTime.now();
|
||||||
|
|
||||||
|
result = this;
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To string.
|
||||||
|
*
|
||||||
|
* @return the string
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public String toString()
|
||||||
|
{
|
||||||
|
String result;
|
||||||
|
|
||||||
|
result = format();
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.util;
|
package fr.devinsy.logar.util;
|
||||||
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.util;
|
package fr.devinsy.logar.util;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.util;
|
package fr.devinsy.logar.util;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.util;
|
package fr.devinsy.logar.util;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.util;
|
package fr.devinsy.logar.util;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.File;
|
import java.io.File;
|
|
@ -16,7 +16,7 @@
|
||||||
* You should have received a copy of the GNU Affero General Public License
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
* along with Logar. If not, see <http://www.gnu.org/licenses/>.
|
||||||
*/
|
*/
|
||||||
package org.april.logar.app;
|
package fr.devinsy.logar.app;
|
||||||
|
|
||||||
import org.apache.log4j.BasicConfigurator;
|
import org.apache.log4j.BasicConfigurator;
|
||||||
import org.apache.log4j.Level;
|
import org.apache.log4j.Level;
|
Loading…
Reference in a new issue