Apache Karaf Decanter 2.x - Documentation

karaf@root()> feature:install decanter-collector-log

karaf@root()> feature:install decanter-collector-log-socket

#
# Decanter Log/Log4j Socket collector configuration
#

#port=4560
#workers=10

karaf@root()> feature:install decanter-collector-file

type=my
path=/path/to/file
any=value

regex=(.*foo.*)

type=karaf-log-file
path=/path/to/karaf/data/log/karaf.log
regex=(.*my.*)
my=stuff

karaf@root()> feature:install decanter-collector-eventadmin

event.topics=my/*

karaf@root()> feature:install decanter-collector-jmx

#
# Decanter Local JMX collector configuration
#

# Name/type of the JMX collection
type=jmx-local

# URL of the JMX MBeanServer.
# local keyword means the local platform MBeanServer or you can specify to full JMX URL
# like service:jmx:rmi:///jndi/rmi://hostname:port/karaf-instance
url=local

# Username to connect to the JMX MBeanServer
#username=karaf

# Password to connect to the JMX MBeanServer
#password=karaf

# Object name filter to use. Instead of harvesting all MBeans, you can select only
# some MBeans matching the object name filter
#object.name=org.apache.camel:context=*,type=routes,name=*

# Several object names can also be specified.
# What matters is that the property names begin with "object.name".
#object.name.system=java.lang:*
#object.name.karaf=org.apache.karaf:type=http,name=*
#object.name.3=org.apache.activemq:*

# You can also execute operations on some MBeans, providing the object name, operation, arguments (separated by ,)
# and signatures (separated by ,) for the arguments (separated by |)
#operation.name.rootLogger=org.apache.karaf:type=log,name=root|getLevel|rootLogger|java.lang.String

karaf@root()> feature:install decanter-collector-jmx-activemq

#
# Decanter Local ActiveMQ JMX collector configuration
#

# Name/type of the JMX collection
type=jmx-activemq

# URL of the JMX MBeanServer.
# local keyword means the local platform MBeanServer or you can specify to full JMX URL
# like service:jmx:rmi:///jndi/rmi://hostname:port/karaf-instance
url=local

# Username to connect to the JMX MBeanServer
#username=karaf

# Password to connect to the JMX MBeanServer
#password=karaf

# Object name filter to use. Instead of harvesting all MBeans, you can select only
# some MBeans matching the object name filter
object.name=org.apache.activemq:*

karaf@root()> feature:install decanter-collector-jmx-camel

#
# Decanter Local Camel JMX collector configuration
#

# Name/type of the JMX collection
type=jmx-camel

# URL of the JMX MBeanServer.
# local keyword means the local platform MBeanServer or you can specify to full JMX URL
# like service:jmx:rmi:///jndi/rmi://hostname:port/karaf-instance
url=local

# Username to connect to the JMX MBeanServer
#username=karaf

# Password to connect to the JMX MBeanServer
#password=karaf

# Object name filter to use. Instead of harvesting all MBeans, you can select only
# some MBeans matching the object name filter
object.name=org.apache.camel:context=*,type=routes,name=*

karaf@root()> feature:install decanter-collector-oshi

################################################################################
#
#    Licensed to the Apache Software Foundation (ASF) under one or more
#    contributor license agreements.  See the NOTICE file distributed with
#    this work for additional information regarding copyright ownership.
#    The ASF licenses this file to You under the Apache License, Version 2.0
#    (the "License"); you may not use this file except in compliance with
#    the License.  You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
################################################################################

#
# Decanter oshi (system) collector
#

# computerSystem=true
# computerSystem.baseboard=true
# computerSystem.firmware=true
# memory=true
# processors=true
# processors.logical=true
# displays=true
# disks=true
# disks.partitions=true
# graphicsCards=true
# networkIFs=true
# powerSources=true
# soundCards=true
# sensors=true
# usbDevices=true
# operatingSystem=true
# operatingSystem.fileSystems=true
# operatingSystem.networkParams=true
# operatingSystem.processes=true
# operatingSystem.services=true

karaf@root()> feature:install decanter-collector-system

#
# Decanter OperationSystem Collector configuration
#

# This collector executes system commands, retrieve the exec output/err
# sent to the appenders
#
# You can define the number of thread to use for parallelization command calls:
# thread.number=1
#
# The format is command.key=command_to_execute
# where command is a reserved keyword used to identify a command property
# for instance:
#
# command.df=df -h
# command.free=free
#
# You can also create a script containing command like:
#
#   df -k / | awk -F " |%" '/dev/{print $8}'
#
# This script will get the available space on the / filesystem for instance.
# and call the script:
#
# command.df=/bin/script
#
# Another example of script to get the temperature:
#
#   sensors|grep temp1|awk '{print $2}'|cut -b2,3,4,5
#

command.name=system_command

command.df=df -k / | awk -F " |%" '/dev/{print $8}'

karaf@root()> feature:install decanter-collector-socket

# Decanter Socket Collector

# Port number on which to listen
#port=34343

# Number of worker threads to deal with
#workers=10

# Protocol tcp(default) or udp
#protocol=tcp

# Unmarshaller to use
# Unmarshaller is identified by data format. The default is json, but you can use another unmarshaller
unmarshaller.target=(dataFormat=json)

karaf@root()> feature:install decanter-collector-jms

######################################
# Decanter JMS Collector Configuration
######################################

# Name of the JMS connection factory
connection.factory.name=jms/decanter

# Name of the destination
destination.name=decanter

# Type of the destination (queue or topic)
destination.type=queue

# Connection username
# username=

# Connection password
# password=

karaf@root()> feature:install decanter-collector-mqtt

#######################################
# Decanter MQTT Collector Configuration
#######################################

# URI of the MQTT broker
server.uri=tcp://localhost:61616

# MQTT Client ID
client.id=decanter

# MQTT topic name
topic=decanter

karaf@root()> feature:install decanter-collector-kafka

###############################
# Decanter Kafka Configuration
###############################

# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster
#bootstrap.servers=localhost:9092

# An id string to identify the group where the consumer belongs to
#group.id=decanter

# Enable auto commit of consumed messages
#enable.auto.commit=true

# Auto commit interval (in ms) triggering the commit
#auto.commit.interval.ms=1000

# Timeout on the consumer session
#session.timeout.ms=30000

# Serializer class for key that implements the Serializer interface
#key.serializer=org.apache.kafka.common.serialization.StringSerializer

# Serializer class for value that implements the Serializer interface.
#value.serializer=org.apache.kafka.common.serialization.StringSerializer

# Name of the topic
#topic=decanter

# Security (SSL)
#security.protocol=SSL

# SSL truststore location (Kafka broker) and password
#ssl.truststore.location=${karaf.etc}/keystores/keystore.jks
#ssl.truststore.password=karaf

# SSL keystore (if client authentication is required)
#ssl.keystore.location=${karaf.etc}/keystores/clientstore.jks
#ssl.keystore.password=karaf
#ssl.key.password=karaf

# (Optional) SSL provider (default uses the JVM one)
#ssl.provider=

# (Optional) SSL Cipher suites
#ssl.cipher.suites=

# (Optional) SSL Protocols enabled (default is TLSv1.2,TLSv1.1,TLSv1)
#ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1

# (Optional) SSL Truststore type (default is JKS)
#ssl.truststore.type=JKS

# (Optional) SSL Keystore type (default is JKS)
#ssl.keystore.type=JKS

# Security (SASL)
# For SASL, you have to configure Java System property as explained in http://kafka.apache.org/documentation.html#security_ssl

karaf@root()> feature:install decanter-collector-rest-servlet

karaf@root()> feature:install decanter-collector-soap

#
# Decanter SOAP collector
#

url=http://localhost:8080/cxf/service
soap.request=

karaf@root()> feature:install decanter-collector-dropwizard

karaf@root()> feature:install decanter-collector-jdbc

#######################################
# Decanter JDBC Collector Configuration
#######################################

# DataSource to use
dataSource.target=(osgi.jndi.service.name=jdbc/decanter)

# SQL Query to retrieve data
query=select * from TABLE

karaf@root()> feature:install decanter-collector-configadmin

karaf@root()> feature:install decanter-collector-prometheus

prometheus.url=http://host/prometheus

karaf@root()> feature:install decanter-collector-redis

address=localhost:6379

#
# Define the connection mode.
# Possible modes: Single (default), Master_Slave, Sentinel, Cluster
#
mode=Single

#
# Name of the Redis map
# Default is Decanter
#
map=Decanter

#
# For Master_Slave mode, we define the location of the master
# Default is localhost:6379
#
#masterAddress=localhost:6379

#
# For Sentinel model, define the name of the master
# Default is myMaster
#
#masterName=myMaster

#
# For Cluster mode, define the scan interval of the nodes in the cluster
# Default value is 2000 (2 seconds).
#
#scanInterval=2000

#
# Key pattern to looking for.
# Default is *
#
#keyPattern=*

karaf@root()> feature:install decanter-collector-elasticsearch

# HTTP address of the elasticsearch nodes (separated with comma)
addresses=http://localhost:9200

# Basic username and password authentication (no authentication by default)
#username=user
#password=password

# Name of the index to request (decanter by default)
#index=decanter

# Query to request document (match all by default)
#query=

# Starting point for the document query (no from by default)
#from=

# Max number of documents retrieved (no max by default)
#max=

# Search timeout, in seconds (no timeout by default)
#timeout=

karaf@root()> feature:install decanter-appender-log

karaf@root()> feature:install decanter-appender-elasticsearch

karaf@root()> feature:install decanter-appender-file

karaf@root()> feature:install decanter-appender-jdbc

#######################################
# Decanter JDBC Appender Configuration
#######################################

# Name of the JDBC datasource
datasource.name=jdbc/decanter

# Name of the table storing the collected data
table.name=decanter

# Dialect (type of the database)
# The dialect is used to create the table
# Supported dialects are: generic, derby, mysql
# Instead of letting Decanter created the table, you can create the table by your own
dialect=generic

karaf@root()> feature:install decanter-appender-jms

#####################################
# Decanter JMS Appender Configuration
#####################################

# Name of the JMS connection factory
connection.factory.name=jms/decanter

# Name of the destination
destination.name=decanter

# Type of the destination (queue or topic)
destination.type=queue

# Connection username
# username=

# Connection password
# password=

karaf@root()> feature:install decanter-appender-camel

#
# Decanter Camel appender configuration
#

# The destination.uri contains the URI of the Camel endpoint
# where Decanter sends the collected data
destination.uri=direct-vm:decanter

destination.uri=direct-vm:decanter

<?xml version="1.0" encoding="UTF-8"?>
<blueprint xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0">

  <camelContext xmlns="http://camel.apache.org/schema/blueprint">
    <route id="decanter">
      <from uri="direct-vm:decanter"/>
      ...
      ANYTHING
      ...
    </route>
  </camelContext>

</blueprint>

karaf@root()> feature:install decanter-appender-kafka

##################################
# Decanter JMS Kafka Configuration
##################################

# A list of host/port pairs to use for establishing the initial connection to the Kafka cluster
#bootstrap.servers=localhost:9092

# An id string to pass to the server when making requests
# client.id

# The compression type for all data generated by the producer
# compression.type=none

# The number of acknowledgments the producer requires the leader to have received before considering a request complete
# - 0: the producer doesn't wait for ack
# - 1: the producer just waits for the leader
# - all: the producer waits for leader and all followers (replica), most secure
# acks=all

# Setting a value greater than zero will cause the client to resend any record whose send fails with a potentially transient error
# retries=0

# The producer will attempt to batch records together into fewer requests whenever multiple records are being sent to the same partition
# batch.size=16384

# The total bytes of memory the producer can use to buffer records waiting to be sent to the server.
# If records are sent faster than they can be delivered to the server the producer will either block or throw an exception
# buffer.memory=33554432

# Serializer class for key that implements the Serializer interface
# key.serializer=org.apache.kafka.common.serialization.StringSerializer

# Serializer class for value that implements the Serializer interface.
# value.serializer=org.apache.kafka.common.serialization.StringSerializer

# Producer request timeout
# request.timeout.ms=5000

# Max size of the request
# max.request.size=2097152

# Name of the topic
# topic=decanter

# Security (SSL)
# security.protocol=SSL

# SSL truststore location (Kafka broker) and password
# ssl.truststore.location=${karaf.etc}/keystores/keystore.jks
# ssl.truststore.password=karaf

# SSL keystore (if client authentication is required)
# ssl.keystore.location=${karaf.etc}/keystores/clientstore.jks
# ssl.keystore.password=karaf
# ssl.key.password=karaf

# (Optional) SSL provider (default uses the JVM one)
# ssl.provider=

# (Optional) SSL Cipher suites
# ssl.cipher.suites=

# (Optional) SSL Protocols enabled (default is TLSv1.2,TLSv1.1,TLSv1)
# ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1

# (Optional) SSL Truststore type (default is JKS)
# ssl.truststore.type=JKS

# (Optional) SSL Keystore type (default is JKS)
# ssl.keystore.type=JKS

# Security (SASL)
# For SASL, you have to configure Java System property as explained in http://kafka.apache.org/documentation.html#security_ssl

karaf@root()> feature:install decanter-appender-redis

#######################################
# Decanter Redis Appender Configuration
#######################################

#
# Location of the Redis broker
# It's possible to use a list of brokers, for instance:
# host= locahost:6389,localhost:6332,localhost:6419
#
# Default is localhost:6379
#
address=localhost:6379

#
# Define the connection mode.
# Possible modes: Single (default), Master_Slave, Sentinel, Cluster
#
mode=Single

#
# Name of the Redis map
# Default is Decanter
#
map=Decanter

#
# For Master_Slave mode, we define the location of the master
# Default is localhost:6379
#
#masterAddress=localhost:6379

#
# For Sentinel model, define the name of the master
# Default is myMaster
#
#masterName=myMaster

#
# For Cluster mode, define the scan interval of the nodes in the cluster
# Default value is 2000 (2 seconds).
#
#scanInterval=2000

karaf@root()> feature:install decanter-appender-mqtt

#server=tcp://localhost:9300
#clientId=decanter
#topic=decanter

karaf@root()> feature:install decanter-appender-cassandra

###########################################
# Decanter Cassandra Appender Configuration
###########################################

# Name of Keyspace
keyspace.name=decanter

# Name of table to write to
table.name=decanter

# Cassandra host name
cassandra.host=

# Cassandra port
cassandra.port=9042

karaf@root()> feature:install decanter-appender-influxdb

##########################################
# Decanter InfluxDB Appender Configuration
##########################################

# URL of the InfluxDB database
url=

# InfluxDB server username
#username=

# InfluxDB server password
#password=

# InfluxDB database name
database=decanter

karaf@root()> feature:install decanter-appender-mongodb

################################
# Decanter MongoDB Configuration
################################

# MongoDB connection URI
#uri=mongodb://localhost

# MongoDB database name
#database=decanter

# MongoDB collection name
#collection=decanter

karaf@root()> feature:install decanter-appender-socket

# Decanter Socket Appender

# Hostname (or IP address) where to send the collected data
#host=localhost

# Port number where to send the collected data
#port=34343

karaf@root()> feature:install decanter-appender-dropwizard

docker run -d --name timescaledb -p 5432:5432 -e POSTGRES_PASSWORD=decanter -e POSTGRES_USER=decanter -e POSTGRES_DATABASE=decanter timescale/timescaledb

rule.my="{'condition':'message:*','level':'ERROR'}

{
    'condition':'QUERY',
    'level':'LEVEL',
    'period':'PERIOD',
    'recoverable':true|false
}

karaf@root()> feature:install decanter-processor-passthrough

karaf@root()> feature:install decanter-processor-aggregate

period=120 # this is the period in seconds
target.topics=decanter/process/aggregate # that's the default target topic

0.foo=first
0.other=bar
1.foo=second
1.other=bar

overwrite=true

package org.apache.karaf.decanter.sample.collector;

import org.osgi.framework.SynchronousBundleListener;
import org.osgi.service.event.EventAdmin;
import org.osgi.service.event.Event;
import java.util.HashMap;

public class BundleCollector implements SynchronousBundleListener {

    private EventAdmin dispatcher;

    public BundleCollector(Event dispatcher) {
      this.dispatcher = dispatcher;
    }

    @Override
    public void bundleChanged(BundleEvent bundleEvent) {
      HashMap<String, Object> data = new HashMap<>();
      data.put("type", "bundle");
      data.put("change", bundleEvent.getType());
      data.put("id", bundleEvent.getBundle().getId());
      data.put("location", bundleEvent.getBundle().getLocation());
      data.put("symbolicName", bundleEvent.getBundle().getSymbolicName());
      Event event = new Event("decanter/collect/bundle", data);
      dispatcher.postEvent(event);
    }

}

package org.apache.karaf.decanter.sample.collector;

import org.osgi.framework.BundleActivator;
import org.osgi.framework.BundleContext;
import org.osgi.framework.ServiceReference;
import org.osgi.framework.ServiceRegistration;
import org.osgi.service.event.EventAdmin;
import org.osgi.util.tracker.ServiceTracker;

public class Activator implements BundleActivator {

       private BundleCollector collector;

       public void start(final BundleContext bundleContext) {
           ServiceTracker tracker = new ServiceTracker(bundleContext, EventAdmin.class.getName(), null);
           EventAdmin dispatcher = (EventAdmin) tracker.waitForService(10000);
           collector = new BundleCollector(dispatcher);
       }

       public void stop(BundleContext bundleContext) {
           collector = null;
       }

}

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <!--

        Licensed to the Apache Software Foundation (ASF) under one or more
        contributor license agreements.  See the NOTICE file distributed with
        this work for additional information regarding copyright ownership.
        The ASF licenses this file to You under the Apache License, Version 2.0
        (the "License"); you may not use this file except in compliance with
        the License.  You may obtain a copy of the License at

           http://www.apache.org/licenses/LICENSE-2.0

        Unless required by applicable law or agreed to in writing, software
        distributed under the License is distributed on an "AS IS" BASIS,
        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
        See the License for the specific language governing permissions and
        limitations under the License.
    -->

    <modelVersion>4.0.0</modelVersion>

    <groupId>org.apache.karaf.decanter.sample.collector</groupId>
    <artifactId>org.apache.karaf.decanter.sample.collector.bundle</artifactId>
    <version>1.0.0-SNAPSHOT</version>
    <packaging>bundle</packaging>
    <name>Apache Karaf :: Decanter :: Sample :: Collector :: Bundle</name>

    <dependencies>

        <!-- OSGi -->
        <dependency>
            <groupId>org.osgi</groupId>
            <artifactId>org.osgi.core</artifactId>
            <version>4.3.1</version>
        </dependency>
        <dependency>
            <groupId>org.osgi</groupId>
            <artifactId>org.osgi.compendium</artifactId>
            <version>4.3.1</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.felix</groupId>
                <artifactId>maven-bundle-plugin</artifactId>
                <version>2.4.0</version>
                <inherited>true</inherited>
                <extensions>true</extensions>
                <configuration>
                    <instructions>
                        <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
                        <Bundle-Version>${project.version}</Bundle-Version>
                        <Bundle-Activator>org.apache.karaf.decanter.sample.collector.bundle.Activator</Bundle-Activator>
                        <Import-Package>
                            *
                        </Import-Package>
                    </instructions>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>

package org.apache.karaf.decanter.sample.collector.hello;

import org.osgi.service.event.Event;
import org.osgi.service.event.EventAdmin;
import java.util.HashMap;

public class HelloCollector implements Runnable {

  private EventAdmin dispatcher;

  public HelloCollector(EventAdmin dispatcher) {
    this.dispatcher = dispatcher;
  }

  @Override
  public void run() {
    HashMap<String, Object> data = new HashMap<>();
    data.put("type", "hello");
    data.put("message", "Hello World");
    Event event = new Event("decanter/collect/hello", data);
    dispatcher.postEvent(event);
  }

}

package org.apache.karaf.decanter.sample.collector.hello;

import org.osgi.framework.*;
import org.osgi.service.event.EventAdmin;
import org.osgi.util.tracker.ServiceTracker;

public class Activator implements BundleActivator {

    private ServiceRegistration registration;

    public void start(BundleContext bundleContext) {
       ServiceTracker tracker = new ServiceTracker(bundleContext, EventAdmin.class.getName(), null);
       EventAdmin dispatcher = tracker.waitForService(10000);
       HelloCollector collector = new HelloCollector(dispatcher);

       Dictionary<String, String> serviceProperties = new Hashtable<String, String>();
       serviceProperties.put("decanter.collector.name", "hello");
       registration = bundleContext.registerService(Runnable.class, collector, serviceProperties);
    }

    public void stop(BundleContext bundleContext) {
       if (registration != null) registration.unregister();
    }

}

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <!--

        Licensed to the Apache Software Foundation (ASF) under one or more
        contributor license agreements.  See the NOTICE file distributed with
        this work for additional information regarding copyright ownership.
        The ASF licenses this file to You under the Apache License, Version 2.0
        (the "License"); you may not use this file except in compliance with
        the License.  You may obtain a copy of the License at

           http://www.apache.org/licenses/LICENSE-2.0

        Unless required by applicable law or agreed to in writing, software
        distributed under the License is distributed on an "AS IS" BASIS,
        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
        See the License for the specific language governing permissions and
        limitations under the License.
    -->

    <modelVersion>4.0.0</modelVersion>

    <groupId>org.apache.karaf.decanter.sample.collector</groupId>
    <artifactId>org.apache.karaf.decanter.sample.collector.hello</artifactId>
    <version>1.0.0-SNAPSHOT</version>
    <packaging>bundle</packaging>
    <name>Apache Karaf :: Decanter :: Sample :: Collector :: Hello</name>

    <dependencies>

        <!-- OSGi -->
        <dependency>
            <groupId>org.osgi</groupId>
            <artifactId>org.osgi.core</artifactId>
            <version>4.3.1</version>
        </dependency>
        <dependency>
            <groupId>org.osgi</groupId>
            <artifactId>org.osgi.compendium</artifactId>
            <version>4.3.1</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.felix</groupId>
                <artifactId>maven-bundle-plugin</artifactId>
                <version>2.4.0</version>
                <inherited>true</inherited>
                <extensions>true</extensions>
                <configuration>
                    <instructions>
                        <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
                        <Bundle-Version>${project.version}</Bundle-Version>
                        <Bundle-Activator>org.apache.karaf.decanter.sample.collector.hello.Activator</Bundle-Activator>
                        <Import-Package>
                            *
                        </Import-Package>
                    </instructions>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>