description: Pipeline for parsing apache error logs
processors:
- set:
    field: event.ingested
    value: '{{_ingest.timestamp}}'
- rename:
    field: message
    target_field: event.original
- grok:
    field: event.original
    patterns:
    - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client
      %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}
    - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\]
      \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\](
      \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}
    pattern_definitions:
      # Apache log level can have numeric sub-levels such as trace1.
      APACHE_LOGLEVEL: '%{LOGLEVEL}[0-9]*'
      APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}'
    ignore_missing: true
- grok:
    field: message
    patterns:
    - "File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}"
    - "File does not exist: %{URIPATH:file.path}"
    ignore_missing: true
    ignore_failure: true
- date:
    if: ctx.event.timezone == null
    field: apache.error.timestamp
    target_field: '@timestamp'
    formats:
    - EEE MMM dd H:m:s yyyy
    - EEE MMM dd H:m:s.SSSSSS yyyy
    on_failure:
    - append:
        field: error.message
        value: '{{ _ingest.on_failure_message }}'
- date:
    if: ctx.event.timezone != null
    field: apache.error.timestamp
    target_field: '@timestamp'
    formats:
    - EEE MMM dd H:m:s yyyy
    - EEE MMM dd H:m:s.SSSSSS yyyy
    timezone: '{{ event.timezone }}'
    on_failure:
    - append:
        field: error.message
        value: '{{ _ingest.on_failure_message }}'
- remove:
    field:
      - apache.error.timestamp
      - _tmp.url_orig
    ignore_failure: true
- set:
    field: event.kind
    value: event
- set:
    field: event.category
    value: web
- script:
    if: "ctx?.log?.level != null"
    lang: painless
    source: >-
      def err_levels = ["emerg", "alert", "crit", "error", "warn"];
      if (err_levels.contains(ctx.log.level)) {
        ctx.event.type = "error";
      } else {
        ctx.event.type = "info";
      }

- grok:
    field: source.address
    ignore_missing: true
    patterns:
    - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
- geoip:
    field: source.ip
    target_field: source.geo
    ignore_missing: true
- geoip:
    database_file: GeoLite2-ASN.mmdb
    field: source.ip
    target_field: source.as
    properties:
    - asn
    - organization_name
    ignore_missing: true
- rename:
    field: source.as.asn
    target_field: source.as.number
    ignore_missing: true
- rename:
    field: source.as.organization_name
    target_field: source.as.organization.name
    ignore_missing: true
- script:
    lang: painless
    description: This script processor iterates over the whole document to remove fields with null values.
    source: |
      void handleMap(Map map) {
        for (def x : map.values()) {
          if (x instanceof Map) {
              handleMap(x);
          } else if (x instanceof List) {
              handleList(x);
          }
        }
        map.values().removeIf(v -> v == null);
      }
      void handleList(List list) {
        for (def x : list) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
        }
      }
      handleMap(ctx);
on_failure:
- set:
    field: error.message
    value: '{{ _ingest.on_failure_message }}'
