Collection Agent configuration reference

# Collection Agent Configuration Reference

# Directory containing plugin artifacts. Required.
pluginDirectory: /usr/local/lib/geneos/plugins

# Agent monitoring and selfMetrics.
# This section is optional.
monitoring:

  # Optional. Defaults to true.
  enabled: true

  # Health and metrics reporting interval in milliseconds. Defaults to 10 seconds.
  reportingInterval: 10000

  # The agent will listen on an HTTP port so that an external system can probe its health.
  # In Kubernetes, this can be used in conjunction with the readiness/liveness probes.
  # 200 is returned if the agent is started, 500 otherwise.
  healthProbe:

    # Optional. Defaults to true.
    enabled: true

    # HTTP listen port, defaults to 8080.
    listenPort: 8080

  # Agent selfMetrics.
  selfMetrics:

    # Whether to enable self metric collection (optional, defaults to true).
    enabled: true

    # Dimensions to add to all selfMetrics from this agent (optional).
    dimensions:
      custom: value

#
# A collector creates data points and submits them to the workflow.
#
collectors:

  # Collector type (all collectors are of type 'plugin').
  - type: plugin

    # Optional. Defaults to true.
    enabled: true

    # Optional name used in logging.  If omitted, an auto-generated name will be assigned.
    name: statsd

    # Fully qualified or simple className of the collector in the plugin jar.
    className: StatsdServer

    # Data point processors applied to data points published from this collector.
    # This optional processing chain allows for manipulating and/or filtering data points prior
    # to workflow publication. This is the recommended way to perform edge processing, when applicable, so that
    # unneeded data can be dropped before incurring workflow overhead.
    processors:
      # For example, drop all events collected from statsd.  See "workflow -> common -> processors" section for
      # details on each type of processor.
      - type: drop-filter
        matchers:
          - type: kind
            kind: generic-event

    # Additional properties are specific to each collector type.  See plugin's configuration reference for details.
    listenPort: 8125

#
# A reporter receives data points from the workflow and sends them to a remote target.
# At least one reporter must be configured.
#
reporters:

  # Each reporter has these common configuration settings:
  - type: [ logging | tcp | routing | plugin ]

    # Optional. Defaults to true.
    enabled: true

    # Reporter name. Referenced from a pipeline's 'reporter' setting.
    name: myReporterName

    # Persist all data points sent to this reporter.  It's intended for testing purposes only
    # and is disabled by default.
    recording:
      enabled: false

      # Directory where the recording is saved.  If undefined, a directory with the name of the reporter
      # will be created in the current working directory.
      directory: /var/lib/geneos/recording

      # Maximum number of data points to record.  Recording will stop when capacity is reached.
      # Default value shown.
      capacity: 1000000

    # Optional: Enabled store and forward based reporting.
    # This is typically only used when a reporter is being used as a routing reporter destination.
    storeAndForward:
      # Mandatory. Root directory for store and forward persisted messages.
      directory: /var/lib/geneos/collection-agent
      # Optional. Store capacity. Defaults to 8192 (8 Ki messages).
      capacity: 8192
      # Optional. Max store file length in bytes. Defaults to 16777216 (16 MiB).
      maxFileLength: 16777216

  # Logging reporter that simply logs each data point to stdout.  This is intended for testing purposes only.
  - type: logging
    name: logging

    # Log level at which each data point is logged.  Can be: error, info (default), warn, debug or trace.
    level: info

  # TCP reporter that sends data points over a TCP connection.
  - type: tcp
    name: myTcpReporter

    # The TCP server hostname. Default value shown.
    hostname: localhost

    # The TCP server port. Default value shown.
    port: 7137

    # The TCP server connection timeout in milliseconds. Default value shown.
    connectionTimeoutMillis: 10000

    # The TCP server write timeout in milliseconds. Default value shown.
    writeTimeoutMillis: 10000

    # Optional. The TCP server TLS configuration.
    tlsConfig:

      # Whether to enforce server authentication or not. Optional. Defaults to false (server authentication is enforced).
      # If set to true, the trust chain file setting is ignored.
      insecure: false

      # The system default is used when not set.
      trustChainFile: ${env:TCP_TRUST_CHAIN_FILE}

      # The client key. Optional. Used only for mTLS (client authentication).
      keyFile: ${env:TCP_KEY_FILE}

      # The client certificate. Optional. Used only for mTLS (client authentication).
      certFile: ${env:TCP_CERT_FILE}

      # The list of TLS protocols to enable. Optional. Defaults to TLSv1.3 and TLSv1.2 only.
      protocols: [ TLSv1.3, TLSv1.2 ]

  # Routing reporter which can be used to route data points via 1 of N other reporters based on matching criteria.
  # Note that for other reporters to be valid routing destinations they must:
  # - appear in the configuration before this reporter
  # - have a 'name' attribute by which they can be referenced.
  - type: routing
    # Optional but recommended component name.
    name: router
    # Optional. Route eviction timeout. Default is 2 seconds.
    # The router automatically evicts slow routes from the list of alternatives when routing messages for the duration
    # of the 'routeRestorationTimeout'. This is to minimise the impact of slow routes on the system as a whole.
    # It is possible, on a per-route basis, to override this such that specific routes are not evicted regardless of
    # their timing characteristics.
    # A value of 0 disables automatic route eviction completely.
    routeEvictionTimeout: 2000
    # Optional. Timeout for automatically restoring an evicted route. Defaults to 60 seconds.
    routeRestorationTimeout: 60000
    # Optional. Route restoration timeout. Default is 60 seconds.
    # Optional. Do not reject data points that don't match any route. Defaults to false.
    ignoreNoMatchingRoute: false
    # Optional. Routing type: [ first (route only via first matching route) | all (route via all matching routes)].
    # Defaults to 'first'.
    routeType: first
    # List of possible routes.
    # The routes are searched in the order specified for the first match.
    routes:
      # Mandatory. Destination reporter name.
      - reporter: destination-reporter-1
        # Optional. Manually enable/disable this route. Defaults to true.
        enabled: true
        # Optional. Specify that this route is never to be evicted. Defaults to false.
        doNotEvict: false
        # Optional. Set the 'ca_deliver_to_name' property on all data points on all matched routes, which is carried on
        # the wire, that is designed to be used in the multi-hop routing case.
        deliverTo: multihop-reporter
        # Optional. Match condition type: 'any' (logical OR) | 'all' (logical AND) over the list of matchers.
        # Defaults to 'any'.
        match: any
        # Mandatory. List of matchers.
        matchers:
          # Mandatory. The data point field to be matched: [ name | namespace | dimension | property ].
          # If the match type is 'dimension' or 'property' then the 'key' attribute is also required.
          - type: name
            # Matching regular expression.
            pattern: promtest.*
          - type: dimension
            # Mandatory when type is 'dimension' or 'property'. Specifies dimension key containing value to match.
            key: dimension_key
            pattern: dimension_value
      - reporter: destination-reporter-2
        match: any
        matchers:
          - type: name
            pattern: jvm.*
      - reporter: destination-reporter-3
        match: all
        matchers:
          # Will match any data point, to this can be used as a catch-all for any previously unmatched data points.
          - type: name
            pattern: .*
          # Except any that are explicitly excluded (note that match type above is 'all', i.e. logical AND).
          - type: name
            pattern: not_me
            # It is possible to set any matcher to be an exclusive matcher instead of (the default) an inclusive matcher.
            exclude: true
      - reporter: destination-reporter-4
        match: all
        matchers:
          # Match all data points collected by the 'statsd' collector ...
          - type: property
            # The special property 'ca_collector_name' is guaranteed to be available and populated with the name of
            # the CA collector.
            key: ca_collector_name
            pattern: statsd
          # ... Except for internal self-monitoring data points.
          - type: property
            key: ca_collector_name
            # The special collector name 'ca_internal' applies to data points generated internally by the CA itself
            # for selfMonitoring.
            pattern: ca_internal
            exclude: true
      - reporter: destination-reporter-5
        match: any
        matchers:
          # Match any data point of the specified types.
          # Valid data types are:
          #   [entity_attribute | counter | gauge | generic_event | generic_histogram | log_event | status_metric |
          #    signal_event | snooze_event | entity_attribute_group_snapshot | open_telemetry_span_group]
          - type: data-type
            pattern: status_metric
          - type: data-type
            pattern: counter
          - type: data-type
            pattern: gauge

  # Example configuration of a multihop router (i.e. the intermediate link) which matches the 'ca_deliver_to_name'
  # property to route.
  - type: routing
    name: multihop-router
    routeType: first
    routes:
      # By convention, if the reporter name matches the value assigned to the 'ca_deliver_to_name' then it is erased
      # before being delivered to the target reporter and therefore will not appear on the wire.
      - reporter: multihop-reporter
        match: all
        matchers:
          - type: property
            key: ca_deliver_to_name
            pattern: multihop-reporter

  # Target reporter corresponding to the final link in a multihop route (can be any type).
  - type: logging
    name: multihop-reporter

  # External/custom reporters are defined using the 'plugin' type.
  - type: plugin
    name: myCustomReporter

    # Fully qualified or simple class name of the reporter in the plugin jar.
    className: custom-reporter

    # Additional properties are specific to each reporter type.  See plugin's configuration reference for details.
    customProp: asdf

#
# Workflow settings for controlling the flow of data points from plugins to reporters.
# This section is optional - default settings are used if omitted.
#
workflow:
  # Directory to store pipeline persistence.
  # Required only if at least one pipeline uses 'disk' store type.
  # The directory must be writable.
  storeDirectory: /var/lib/geneos/collection-agent

  # Optionally override the default validations applied to all data points before ingesting.
  #
  # Validation runs after all configured processors have been applied and just before writing the data point to the
  # pipeline store.
  #
  # The class must implement the WorkflowDataPointValidator interface and may extend DefaultWorkflowDataPointValidator
  # in order to retain the defaults.
  #
  # The value can be the fully qualified or simple class name.
  validatorClass: com.example.MyValidator

  # Pipelines.
  #
  # A pipeline exists for each class of data (metrics/logs/events/attributes/traces)
  #
  # Each pipeline is enabled by default if omitted from the configuration.
  #
  # At least one pipeline must be enabled.  A runtime error will occur if a plugin attempts delivery to a pipeline
  # that is not configured.
  #

  # Metrics pipeline.
  metrics:

    # Reporter to which all data points on this pipeline are sent.
    # This property is optional if there is only one reporter configured.  Otherwise the value is required and
    # must correspond to the 'name' of a reporter defined above.
    reporter: logging

    # Optional. Defaults to true.
    enabled: true

    # Optional. Whether internal resources are pooled or not. Defaults to false. Does not apply in pass-through mode.
    # Resource pools consume more static memory but result in less garbage collection and therefore less CPU load.
    pooling: false

    # Optional. Whether to use file level locking to prevent multiple processes operating over a single persistence
    # store. Defaults to false.
    # When this option is set a second collection agent misconfigured to load the same persistence store as an
    # already running collection agent will fail to start.
    # Lock files are cleared on termination except in the case of abrupt termination (e.g. via SIGKILL) in which case
    # there is a possibility of stale lock files remaining beyond process exit. This situation requires manual
    # intervention to remove the lock files, or the disabling of this option, prior to the next instance being started.
    fileLocking: false

    # Number of retries after initial delivery fails.  Defaults to 3.  For infinite retries set to -1.
    # The interval between consecutive retries for the same message increases from 1 second up to 120 seconds.
    maxRetries: 3

    # Optional pass through mode configuration (disabled by default).
    #
    # In pass through mode, there is no buffering between collectors and the pipeline - data points pass through the
    # pipeline on the thread of the collector. This means that collector threads are directly coupled to the behavior of
    # the eventual reporter. The nature of the coupling is determined by whether the reporter is synchronous or
    # asynchronous and whether pipeline retries are enabled. It is therefore possible the a collector thread becomes
    # blocked awaiting reporter completion.
    passThrough:
      # Optional. Defaults to false.
      enabled: false

      # Optional. Enable fire and forget mode (disabled by default) on this pipeline.
      # Only applicable when passThrough mode is enabled and maxRetries is 0.
      #
      # This option can be used to achieve higher throughput when best effort reporting (i.e. no failure notifications
      # or retries) is an acceptable tradeoff.
      fireAndForget: false

      # Optional. Only applies when the workflow is in 'passThrough' mode. Defaults to 'parallel'.
      #
      # Defines whether multiple threads are allowed to enter the pipeline concurrently or not.
      # Serial mode can be used in the case of stateful pipeline processors.
      concurrency: [ parallel | serial ]

    # Store settings.
    #
    # Data points are stored either in memory or on disk before delivery to a reporter.
    #
    # If a reporter's target becomes unavailable, data points are queued until either the store is full or
    # the reporter target becomes available again.
    #
    # Plugins are informed when a store becomes full and are free to handle the situation in a way that makes
    # sense for that plugin (i.e. dropping the message if not critical, or waiting for the store to re-open before
    # collecting any more data).
    store:

      # Store type.
      #
      # Permitted values:
      # 'memory':  A circular, fixed-size, in-memory store that provides no persistence.  The oldest data point
      #            is removed when adding to a full store, therefore this store never rejects new data points
      #            and will begin to drop data if a slow reporter cannot keep up.
      #
      # 'disk':    A fixed-size store that is persisted to disk.  Requires the workflow 'storeDirectory' setting
      #            to be configured.
      #
      # For the metrics pipeline, it is recommended (and the default) to use a memory store, as metric data is
      # generally non-critical and loses relevance if delayed.
      #
      type: memory

      # Maximum number of data points to hold before the store is considered full and new data points are rejected.
      # The default capacity for a memory store is 8192 data points and 10,000,000 data points for a disk store.
      capacity: 8192

    # Custom processing of data points on this pipeline.  Processors can manipulate, enrich and/or filter
    # data points before reporting.
    #
    # See the 'common' pipeline for more details.
    processors:
      - type: enrichment
        name: metrics-enricher
        dimensions:
          custom_dimension: value

  # Logs pipeline.
  logs:
    reporter: logging
    store:
      # For logs, it is recommended (and the default) to use a disk store if data loss is not tolerable.
      type: disk

      # Maximum size (in bytes) of one store file. Only applicable when store type is "disk".
      # The value must be a multiple of 4096.
      # Optional - default value is 128MB for logs and 16MB for events.
      maxFileLength: 134217728

    # For logs, it is recommended (and the default) to retry infinitely if data loss is not tolerable.
    maxRetries: -1

  # Events pipeline.
  events:
    reporter: logging
    store:
      # For events, it is recommended (and the default) to use a disk store if data loss is not tolerable.
      type: disk

    # For events, it is recommended (and the default) to retry infinitely if data loss is not tolerable.
    maxRetries: -1

  # Attributes pipeline.
  attributes:
    reporter: logging
    store:
      # For attributes, it is recommended (and the default) to use a disk store if data loss is not tolerable.
      type: disk

    # For attributes, it is recommended (and the default) to retry infinitely if data loss is not tolerable.
    maxRetries: -1

  # Traces pipeline.
  # Note: This is an EXPERIMENTAL feature and is not yet generally compatible with most reporters.
  traces:
    reporter: logging
    store:
      # For attributes, it is recommended (and the default) to use a disk store if data loss is not tolerable.
      type: disk

    # For attributes, it is recommended (and the default) to retry infinitely if data loss is not tolerable.
    maxRetries: -1

  # Common pipeline.
  #
  # This is a unique pipeline that only has dataPoint processors (there is no reporter). The processors are applied
  # to data points on all pipelines, before any pipeline-specific processors are applied.
  common:
    # Optional. As for the 'pooling' option for the individual pipelines.
    # Can be used as a shortcut to apply to all pipelines.
    # Can also be overridden by individual pipelines.
    pooling: false

    # Optional. As for the 'fileLocking' option for the individual pipelines.
    # Can be used as a shortcut to apply to all pipelines.
    # Can also be overridden by individual pipelines.
    fileLocking: false

    # dataPoint processors.
    #
    # Processors can manipulate, enrich and/or filter data points before reporting.  They are applied before
    # a data point is saved in the pipeline's store.
    #
    processors:

      # Enrichment processor.  Adds dimensions and/or properties to all data points.
      - type: enrichment

        # Optional. Defaults to true.
        enabled: true

        # Optional name used in logging.  If omitted, an auto-generated name will be assigned.
        name: enricher

        # Whether to overwrite an existing dimension or property with the same name (defaults to false)
        overwrite: false

        # Dimensions to add
        dimensions:
          node_name: ${env:NODE_NAME}

        # Properties to add
        properties:
          prop: value

      # Translation processor.
      #
      # Translates:
      # - data point names
      # - dimension and/or property key/values
      #
      - type: translation

        # Translate data point name via a search and replace operation (optional).
        nameTranslation:
          # The search regular expression.
          # The name is not modified unless it matches this pattern.
          # The pattern may contain group captures which may be reference in the 'replace' pattern.
          search: search-pattern

          # The replace regular expression.
          # May contain group references from the 'search' pattern.
          replace: replace-pattern

        # List of dimension and/or property key/value translators (optional).
        keyValueTranslations:

          # First translator.
          #
          # The source key/value.
          - from:
              # Either 'dimension' or 'property'.
              type: dimension

              # Source dimension or property name.
              name: dim1

              # Optional source value search pattern.
              search: search-pattern

              # Whether or not to delete the source key/value (default is true).
              delete: true

            # The target key/value.
            # If the 'from' specifies 'delete' then the 'to' section may be omitted.
            to:
              # Either 'dimension' or 'property'.
              type: property

              # Target dimension or property name.
              name: prop1

              # Optional target value replace pattern
              replace: replace-pattern

              # Whether or not to overwrite the target if it already exists (default is true).
              overwrite: true

          # Second translator.
          - from:
              type: dimension
              name: dim2
              delete: true
            to:
              type: property
              name: prop2
              overwrite: true

      # Drop filter processor.  Drops data points that match the configured criteria.
      - type: drop-filter

        # One or more match criteria.
        # For a data point to be dropped, all configured criteria must match, otherwise the data point
        # will be forwarded. If no matchers are configured, all data points will be forwarded.
        matchers:

          # Match by data point name, either exactly or via regex.
          - type: name

            # Exact match
            name: kubernetes_node_cpu_usage

            # Regex match (only one of 'name' or 'namePattern' can be configured)
            namePattern: kubernetes_.*

          # Match by data point dimension key and either an exact value or a regex pattern.
          - type: dimension
            key: namespace

            # Exact value match
            value: finance

            # Regex match (only one of 'value' or 'valuePattern' can be configured)
            valuePattern: ns.*

          # Match by data point property key and either an exact value or a regex pattern.
          - type: property
            key: someProperty

            # Exact value match
            value: someValue

            # Regex match (only one of 'value' or 'valuePattern' can be configured)
            valuePattern: value.*

          # Match by data point type. Value kinds are: [attribute|counter|gauge|generic-event|log-event|histogram]
          - type: kind
            kind: counter

      # Forward filter processor.  Forwards data points that match the configured criteria.
      # This behaves inversely to "dropFilter" above but is configured identically.
      - type: forward-filter

        # One or more match criteria.
        # For a data point to be forwarded, all configured criteria must match, otherwise the data point
        # will be dropped.  If no matchers are configured, all data points will be dropped.
        # See "dropFilter" for details on each type of matcher.
        matchers:
          - type: name
            pattern: myCounter

      # Normalize processor.  Normalizes dimension names for consistency in subsequent processing and reporting.
      - type: normalize

        # Optional name used in logging.  If omitted, an auto-generated name will be assigned.
        name: normalize

        # Dimension normalization settings.
        dimensions:

          # Default overwrite behavior, can be overridden per mapping.  Defaults to false.
          overwrite: false

          # Dimension mappings.
          mappings:

            # Old dimension name.
            - from: project

              # New dimension name.
              to: namespace

              # Whether to overwrite if a dimension already exists with the same name.  Defaults to parent setting.
              overwrite: false

      # Simple statistics processor. Logs simple statistics on data points received on this workflow pipeline.
      - type: statistics

        # Processor name. Recommended to use a name representative of the pipeline on which the processor is configured.
        name: metric-stats

        # Optional. Forward data points as normal or drop them. Default is false (drop).
        forward: false

        # Optional. Reporting interval in milliseconds. Default is 10000ms.
        reportingInterval: 10000

      # External/custom processors are defined using the 'plugin' type.
      - type: plugin

        # Optional name used in logging.  If omitted, an auto-generated name will be assigned.
        name: kube-enricher

        # Fully qualified or simple class name of the processor in the plugin jar.
        className: KubernetesEnricher

        # Additional properties are specific to each processor type.  See plugin's configuration reference for details.
        customProp: abc