statool/logar
statool/logar
1
0
Fork 0
Code Issues Pull requests Releases Activity

Improved file parameter management.

Browse source
This commit is contained in:
Christian P. MOMON 2024-07-18 13:50:03 +02:00
parent bf0810dd6f
commit 23d9e13787
4 changed files with 207 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/fr/devinsy/logar/app/Logar.java
View file

@ -98,13 +98,15 @@ public final class Logar
anonymizer.loadMapTable(mapFile);
System.out.println("Table size=" + anonymizer.getMapTable().size());
Files files = FilesUtils.searchEndingWith(source, ".log", ".log.gz").keepFileType().removeContaining("-anon.log").sortByName();
Files files = FilesUtils.search(source, LOGFILE_PATTERN).keepFileType().removeContaining("-anon.").sortByName();
logger.info("file count={}", files.size());
for (File file : files)
{
anonymizer.anonymize(file);
}
System.out.println("Final table size: " + anonymizer.getMapTable().size());
if (mapFile != null)
{
System.out.println("Table size=" + anonymizer.getMapTable().size());

17
src/fr/devinsy/logar/app/anonymizer/Anonymizer.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2021-2024 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
@ -89,20 +89,27 @@ public final class Anonymizer
LineParser lineParser = LogFile.getParser(source);
File target;
if (source.getName().endsWith(".log.gz"))
if (source.getName().endsWith(".log"))
{
target = new File(source.getParentFile(), source.getName().replace(".log.gz", "-anon.log.gz"));
target = new File(source.getParentFile(), source.getName().replaceAll(".log$", "-anon.log."));
}
else
{
target = new File(source.getParentFile(), source.getName().replace(".log", "-anon.log"));
target = new File(source.getParentFile(), source.getName().replace(".log.", "-anon.log."));
}
PrintWriter out = null;
try
{
LineIterator iterator = new LineIterator(source);
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
if (source.getName().endsWith(".gz"))
{
out = new PrintWriter(new GZIPOutputStream(new FileOutputStream(target)));
}
else
{
out = new PrintWriter(new FileOutputStream(target));
}
while (iterator.hasNext())
{
String line = iterator.next();

164
src/fr/devinsy/logar/cli/CLIUtils.java Normal file
View file

@ -0,0 +1,164 @@
/*
* Copyright (C) 2020-2023 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of StatoolInfos, simple service statistics tool.
*
* StatoolInfos is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* StatoolInfos is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with StatoolInfos. If not, see <http://www.gnu.org/licenses/>.
*/
package fr.devinsy.logar.cli;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The Class CLIUtils.
*/
public final class CLIUtils
{
private static Logger logger = LoggerFactory.getLogger(CLIUtils.class);
/**
* Instantiates a new statool infos CLI.
*/
private CLIUtils()
{
}
/**
* Checks if is matching.
*
* @param args
* the args
* @param regexps
* the regexps
* @return true, if is matching
*/
public static boolean isMatching(final String[] args, final String... regexps)
{
boolean result;
if ((args.length == 0) && (regexps == null))
{
result = true;
}
else if ((args.length != 0) && (regexps == null))
{
result = false;
}
else if (args.length != regexps.length)
{
result = false;
}
else
{
boolean ended = false;
int index = 0;
result = false;
while (!ended)
{
if (index < args.length)
{
String arg = StringUtils.trim(args[index]);
String regexp = regexps[index];
if (arg.matches(regexp))
{
index += 1;
}
else
{
ended = true;
result = false;
}
}
else
{
ended = true;
result = true;
}
}
}
//
return result;
}
/**
* Checks if is matching ellipsis.
*
* @param args
* the args
* @param regexps
* the regexps
* @return true, if is matching ellipsis
*/
public static boolean isMatchingEllipsis(final String[] args, final String... regexps)
{
boolean result;
if ((args.length == 0) && (regexps == null))
{
result = true;
}
else if ((args.length != 0) && (regexps == null))
{
result = false;
}
else if (args.length < regexps.length)
{
result = false;
}
else
{
boolean ended = false;
int index = 0;
result = false;
while (!ended)
{
if (index < args.length)
{
String arg = StringUtils.trim(args[index]);
String regexp;
if (index < regexps.length)
{
regexp = regexps[index];
}
else
{
regexp = regexps[regexps.length - 1];
}
if (arg.matches(regexp))
{
index += 1;
}
else
{
ended = true;
result = false;
}
}
else
{
ended = true;
result = true;
}
}
}
//
return result;
}
}

115
src/fr/devinsy/logar/cli/LogarCLI.java
View file

@ -1,5 +1,5 @@
/*
* Copyright (C) 2021 Christian Pierre MOMON <christian@momon.org>
* Copyright (C) 2021-2024 Christian Pierre MOMON <christian@momon.org>
*
* This file is part of Logar, simple tool to manage http log files.
*
@ -56,14 +56,14 @@ public final class LogarCLI
message.appendln("Usage:");
message.appendln(" logar [ -h | -help | --help ]");
message.appendln(" logar [ -v | -version | --version ]");
message.appendln(" logar anonymize <fileordirectory> [<mapfile>] anonymize ip and user");
message.appendln(" logar archive [-dry] <source> <target> archive previous month from /var/log/nginx/ tree");
message.appendln(" logar check <fileordirectory> check line format in log files");
message.appendln(" logar checksort <fileordirectory> check sort in log files");
message.appendln(" logar extract <fields> <fileordirectory> extract one or more fields (ip,user,datetime,useragent) from log files");
message.appendln(" logar identify <fileordirectory> display type of log files");
message.appendln(" logar sort <fileordirectory> sort log files by datetime");
message.appendln(" logar testconcate <fileordirectory> test line concate in log files");
message.appendln(" logar anonymize <fileordirectory> [-map <mapfile>] anonymize ip and user");
message.appendln(" logar archive [-dry] <source> <target> archive previous month from /var/log/nginx/ tree");
message.appendln(" logar check <fileordirectory> check line format in log files");
message.appendln(" logar checksort <fileordirectory> check sort in log files");
message.appendln(" logar extract <fields> <fileordirectory> extract one or more fields (ip,user,datetime,useragent) from log files");
message.appendln(" logar identify <fileordirectory> display type of log files");
message.appendln(" logar sort <fileordirectory> sort log files by datetime");
message.appendln(" logar testconcate <fileordirectory> test line concate in log files");
System.out.println(message.toString());
}
@ -80,65 +80,6 @@ public final class LogarCLI
System.out.println(message.toString());
}
/**
* Checks if is matching.
*
* @param args
* the args
* @param regexps
* the regexps
* @return true, if is matching
*/
public static boolean isMatching(final String[] args, final String... regexps)
{
boolean result;
if ((args.length == 0) && (regexps == null))
{
result = true;
}
else if ((args.length != 0) && (regexps == null))
{
result = false;
}
else if (args.length != regexps.length)
{
result = false;
}
else
{
boolean ended = false;
int index = 0;
result = false;
while (!ended)
{
if (index < args.length)
{
String arg = args[index];
String regexp = regexps[index];
if (arg.matches(regexp))
{
index += 1;
}
else
{
ended = true;
result = false;
}
}
else
{
ended = true;
result = true;
}
}
}
//
return result;
}
/**
*
* This method launch CLI.
@ -174,84 +115,84 @@ public final class LogarCLI
{
logger.info("{} Logar call: {}", LocalDateTime.now(), new StringList(args).toStringSeparatedBy(" "));
if (isMatching(args))
if (CLIUtils.isMatching(args))
{
logger.info("No parameter.");
displayHelp();
}
else if (isMatching(args, "(-h|--h|--help)"))
else if (CLIUtils.isMatching(args, "(-h|--h|--help)"))
{
displayHelp();
}
else if (isMatching(args, "(-v|-version|--version)"))
else if (CLIUtils.isMatching(args, "(-v|-version|--version)"))
{
displayVersion();
}
else if (isMatching(args, "anonymize", "\\s*\\S+\\s*"))
{
File source = new File(args[1]);
Logar.anonymize(source);
}
else if (isMatching(args, "anonymize", "\\s*\\S+\\s*", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "anonymize", ".+", "-map", ".+"))
{
File source = new File(args[1]);
File map = new File(args[2]);
Logar.anonymize(source, map);
}
else if (isMatching(args, "archive", "\\s*\\S+\\s*", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatchingEllipsis(args, "anonymize", ".+"))
{
File source = new File(args[1]);
Logar.anonymize(source);
}
else if (CLIUtils.isMatching(args, "archive", ".+", ".+"))
{
File source = new File(args[1]);
File target = new File(args[2]);
Logar.archive(source, target, OnOffOption.OFF);
}
else if (isMatching(args, "archive", "-dry", "\\s*\\S+\\s*", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "archive", "-dry", ".+", ".+"))
{
File source = new File(args[2]);
File target = new File(args[3]);
Logar.archive(source, target, OnOffOption.ON);
}
else if (isMatching(args, "check", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "check", ".+"))
{
File source = new File(args[1]);
Logar.checkLogFiles(source);
}
else if (isMatching(args, "checksort", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "checksort", ".+"))
{
File source = new File(args[1]);
Logar.checkSort(source);
}
else if (isMatching(args, "extract", "\\s*((ip)?(,)?(remoteuser)?(,)?(datetime)?(,)?(useragent)?)\\s*", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "extract", "\\s*((ip)?(,)?(remoteuser)?(,)?(datetime)?(,)?(useragent)?)\\s*", "\\s*\\S+\\s*"))
{
ExtractOptions options = ExtractOptions.of(args[1]);
File source = new File(args[2]);
Logar.extract(source, options);
}
else if (isMatching(args, "identify", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "identify", ".+"))
{
File source = new File(args[1]);
Logar.identify(source);
}
else if (isMatching(args, "sort", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "sort", ".+"))
{
File source = new File(args[1]);
Logar.sort(source);
}
else if (isMatching(args, "statuseragent", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "statuseragent", ".+"))
{
File source = new File(args[1]);
Logar.statUserAgents(source);
}
else if (isMatching(args, "testconcate", "\\s*\\S+\\s*"))
else if (CLIUtils.isMatching(args, "testconcate", ".+"))
{
File source = new File(args[1]);