From 9662d662547707b7a1d4f8fb7ee3dbb5a232cec9 Mon Sep 17 00:00:00 2001 From: "Christian P. MOMON" Date: Tue, 20 Aug 2024 22:57:44 +0200 Subject: [PATCH] Improved main README and resources files. --- README.md | 144 ++++++++++++++-- build-appjar.xml | 8 +- resources/conf/README.md | 1 + resources/conf/sample-web.conf | 14 -- resources/conf/statoolinfos.cron | 3 - resources/etc/apache2/statoolinfos.conf | 5 + resources/etc/cron.d/statoolinfos | 2 + resources/srv/statoolinfos/bin/crawl | 7 + resources/srv/statoolinfos/bin/probe | 7 + resources/srv/statoolinfos/bin/refresh | 10 ++ .../srv/statoolinfos/bin/statoolinfos.sh | 22 +++ resources/srv/statoolinfos/bin/uptime | 7 + .../conf/log4j2-default.properties | 0 .../statoolinfos/conf/sample-federation.conf | 8 + .../statoolinfos/conf/sample-probe-http.conf | 8 + .../conf/sample-probe-minetest.conf | 2 +- .../conf/sample-probe-qrcode.conf} | 2 +- .../srv/statoolinfos/statoolinfos-cron.log | 0 .../statoolinfos/well-known/statoolinfos/BOM | 1 + .../federation-template.properties | 87 ++++++++++ .../organization-template.properties | 136 +++++++++++++++ .../statoolinfos/service-template.properties | 163 ++++++++++++++++++ .../statoolinfos/services.properties | 43 +++++ 23 files changed, 646 insertions(+), 34 deletions(-) create mode 100644 resources/conf/README.md delete mode 100644 resources/conf/sample-web.conf delete mode 100644 resources/conf/statoolinfos.cron create mode 100644 resources/etc/apache2/statoolinfos.conf create mode 100644 resources/etc/cron.d/statoolinfos create mode 100755 resources/srv/statoolinfos/bin/crawl create mode 100755 resources/srv/statoolinfos/bin/probe create mode 100755 resources/srv/statoolinfos/bin/refresh create mode 100755 resources/srv/statoolinfos/bin/statoolinfos.sh create mode 100755 resources/srv/statoolinfos/bin/uptime rename resources/{ => srv/statoolinfos}/conf/log4j2-default.properties (100%) create mode 100644 resources/srv/statoolinfos/conf/sample-federation.conf create mode 100644 resources/srv/statoolinfos/conf/sample-probe-http.conf rename resources/{ => srv/statoolinfos}/conf/sample-probe-minetest.conf (86%) rename resources/{conf/sample-probe-http.conf => srv/statoolinfos/conf/sample-probe-qrcode.conf} (96%) create mode 100644 resources/srv/statoolinfos/statoolinfos-cron.log create mode 100644 resources/srv/statoolinfos/well-known/statoolinfos/BOM create mode 100644 resources/srv/statoolinfos/well-known/statoolinfos/federation-template.properties create mode 100644 resources/srv/statoolinfos/well-known/statoolinfos/organization-template.properties create mode 100644 resources/srv/statoolinfos/well-known/statoolinfos/service-template.properties create mode 100644 resources/srv/statoolinfos/well-known/statoolinfos/services.properties diff --git a/README.md b/README.md index d132710..25c652d 100644 --- a/README.md +++ b/README.md @@ -44,15 +44,43 @@ Read CONCEPT.md and ONTOLOGY.md files. ## INSTALL -Install Java : +Install Java on Debian: ``` apt-get install openjdk-17-jre-headless ``` -Download the last release: https://forge.devinsy.fr/statool/statoolinfos/releases +Download the last release: `https://forge.devinsy.fr/statool/statoolinfos/releases` and unzip the package. -Unzip the package and move the `statoolinfos.jar` and `statoolinfo.sh` files in `/srv/statoolinfos/bin/`. +The `srv/` directory is a example of deployment tree: + +``` +srv/ +srv/statoolinfos +srv/statoolinfos/bin +srv/statoolinfos/bin/refresh +srv/statoolinfos/bin/probe +srv/statoolinfos/bin/uptime +srv/statoolinfos/bin/statoolinfos.sh +srv/statoolinfos/bin/crawl +srv/statoolinfos/conf +srv/statoolinfos/conf/sample-probe-http.conf +srv/statoolinfos/conf/log4j2-default.properties +srv/statoolinfos/conf/sample-probe-minetest.conf +srv/statoolinfos/conf/sample-federation.conf +srv/statoolinfos/conf/sample-probe-qrcode.conf +srv/statoolinfos/cache +srv/statoolinfos/statoolinfos-cron.log +srv/statoolinfos/well-known +srv/statoolinfos/well-known/statoolinfos +srv/statoolinfos/well-known/statoolinfos/organization-template.properties +srv/statoolinfos/well-known/statoolinfos/services.properties +srv/statoolinfos/well-known/statoolinfos/BOM +srv/statoolinfos/well-known/statoolinfos/federation-template.properties +srv/statoolinfos/well-known/statoolinfos/service-template.properties +``` + +Then, move the `statoolinfos.jar` file in `/srv/statoolinfos/bin/`. Create link: @@ -60,6 +88,15 @@ Create link: cd /srv/statoolinfos/bin/ ; ln -s statoolinfos.sh statoolinfos ``` +Edit script in `/srv/statoolinfos/bin/` to target your federation configuration file. + + +In case of bad language output, add these lines on top of cron file: + +``` +LANGUAGE=en_EN.UTF8 +LC_ALL=en_EN.UTF-8 +``` ## Usage @@ -90,7 +127,7 @@ Usage: statoolinfos list errlog display http error log lines ``` -### Simple configuration file +## Simple federation configuration file Example of `libre-service.eu.conf`: @@ -108,32 +145,87 @@ The field `input` gives the federation properties URL file. Reminder, the Statoo The field `cache` gives local directory used as data cache. + ## Crawl -Example of cron configuration: +The `statoolinfos crawl` command loads the federation configuration file, then downloads the federation properties file and download all files set in. + +In case of use with StatoolInfosWeb, never forget to give read permissions to Tomcat server. So, create a dedicated script would help. + +Example of script `/srv/statoolinfos/bin/crawl`: ``` -55 * * * * root /srv/statoolinfos/bin/statoolinfos crawl /srv/statoolinfos/conf/libre-service.eu.conf >> /srv/statoolinfos/statoolinfos-cron.log +#!/bin/bash + +echo "========== CRAWL at $(date --iso-8601=seconds)" +cd $(dirname "$0") +./statoolinfos crawl ../conf/federation.conf +chmod -R g+r /srv/statoolinfos/cache +chgrp -R tomcat /srv/statoolinfos/cache +``` + +Example of cron configuration `/etc/cron.d/statoolinfos`: ``` +4 * * * * root /srv/statoolinfos/bin/crawl >> /srv/statoolinfos/statoolinfos-cron.log +``` + +Adjust your frequency according to your need. In case of large federation, once an hour is a good compromise. ## Uptime -This command is very useful to collect the uptime of services. It's just to share which services are down and since how many time. It is not a monitoring feature because an one hour check is enough for this. Actually, only website are checked. +The `statoolinfos uptime` command is very useful to collect the uptime of services. It's just to share which services are down and since how many time. It is not a monitoring feature because an one hour check is enough for this. Actually, only website are checked. + +In case of use with StatoolInfosWeb, never forget to give read permissions to Tomcat server. So, create a dedicated script would help. + +Example of script `/srv/statoolinfos/bin/uptime`: -Example of cron configuration: ``` -4 * * * * root /srv/statoolinfos/bin/statoolinfos uptime /srv/statoolinfos/conf/libre-service.eu.conf >> /srv/statoolinfos/statoolinfos-cron.log +#!/bin/bash +echo "========== UPTIME at $(date --iso-8601=seconds)" +cd $(dirname "$0") +./statoolinfos uptime ../conf/federation.conf +chmod -R g+r /srv/statoolinfos/cache +chgrp -R tomcat /srv/statoolinfos/cache +``` + +Example of cron configuration `/etc/cron.d/statoolinfos`: + +``` +4 * * * * root /srv/statoolinfos/bin/uptime >> /srv/statoolinfos/statoolinfos-cron.log ``` One check per hour is a good idea. In the uptime page, the mouse over a yellow status icon show the list of the check results. +In case of crawl, uptime and probe cron on the same computer, you can have different cron frequencies. It is better to have only script for call all them: + +``` +#!/bin/bash + +echo "========== REFRESH at $(date --iso-8601=seconds)" +cd $(dirname "$0") +./probe +./crawl +./uptime +#./htmlize +chmod -R g+r /srv/statoolinfos/cache +chgrp -R tomcat /srv/statoolinfos/cache + +``` + +And so your cron can be like that: + +``` +4 0,12,23 * * * root /srv/statoolinfos/bin/refresh >> /srv/statoolinfos/statoolinfos-cron.log +4 1-11,13-22 * * * root /srv/statoolinfos/bin/uptime >> /srv/statoolinfos/statoolinfos-cron.log +``` -## Htmlize removed -The htmlize command is removed since 2024-07. +## Htmlize command **removed** + +The `statoolinfos htmlize` command is removed since 2024-07. As a reminder, the htmlize command generated absolutely all the pages needed for a static website. @@ -142,10 +234,34 @@ As the number of pages generated became too large (> 10000 with 100 organization A dynamic web site allows to generate page only when it is necessary. - ## Probe (Generate metrics files) -### Basics +### Basics +The `statoolinfos probe` command computes data sources to generate metric properties files. In case of already existings file, the data are loaded before the computation to be updated. + +In case of use with StatoolInfosWeb, never forget to give read permissions to Tomcat server. So, create a dedicated script would help. + +Example of script `/srv/statoolinfos/bin/probe`: + +``` +#!/bin/bash + +echo "========== PROBE at $(date --iso-8601=seconds)" +cd $(dirname "$0") +./statoolinfos probe -previousday ../conf/ +chmod -R g+r /srv/statoolinfos/cache +chgrp -R tomcat /srv/statoolinfos/cache +``` + +Note: set a specific configuration file is not necessary, because you can have multiple service to probe. Only probe configuration files will be manage. + +Example of cron configuration `/etc/cron.d/statoolinfos`: + +``` +4 * * * * root /srv/statoolinfos/bin/probe >> /srv/statoolinfos/statoolinfos-cron.log +``` + +### Sample configuration file Create a configuration file `/srv/statoolInfos/conf/foo.bar.org.conf`: @@ -166,7 +282,7 @@ Generate the metric file for the first time: Create a cron file in `/etc/cron.d/statoolinfos` to update the metric file everyday: ``` -1 0,12 * * * root /srv/statoolinfos/bin/statoolinfos probe -previousday /srv/statoolinfos/conf/foo.bar.org.conf >> /srv/statoolinfos/cron.log +4 0,12 * * * root /srv/statoolinfos/bin/statoolinfos probe -previousday /srv/statoolinfos/conf/foo.bar.org.conf >> /srv/statoolinfos/cron.log ``` In case of several configuration files, just set the configuration directory (without wildcard): diff --git a/build-appjar.xml b/build-appjar.xml index 90fa58f..37feb57 100644 --- a/build-appjar.xml +++ b/build-appjar.xml @@ -126,16 +126,22 @@ + + + + + + - +