description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins."

processors:
- set:
    field: event.ingested
    value: '{{_ingest.timestamp}}'
- rename:
    field: message
    target_field: event.original
- grok:
    field: event.original
    patterns:
    - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
      "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?"
      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
      "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?"
      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
      "-" %{NUMBER:http.response.status_code:long} -'
    - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol}
      %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig}
      HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long})
    ignore_missing: true
- uri_parts:
    field: _tmp.url_orig
    ignore_failure: true
- set:
    field: url.domain
    value: "{{destination.domain}}"
    if: ctx.url?.domain == null && ctx.destination?.domain != null
- remove:
    field:
      - _tmp.url_orig
    ignore_missing: true
- set:
    field: event.kind
    value: event
- set:
    field: event.category
    value: web
- set:
    field: event.outcome
    value: success
    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
- set:
    field: event.outcome
    value: failure
    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"
- grok:
    field: source.address
    ignore_missing: true
    patterns:
    - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
- set:
    copy_from: '@timestamp'
    field: event.created
- date:
    field: apache.access.time
    target_field: '@timestamp'
    formats:
    - dd/MMM/yyyy:H:m:s Z
    ignore_failure: true
- remove:
    field: apache.access.time
    ignore_failure: true
- user_agent:
    field: user_agent.original
    ignore_failure: true
- geoip:
    field: source.ip
    target_field: source.geo
    ignore_missing: true
- geoip:
    database_file: GeoLite2-ASN.mmdb
    field: source.ip
    target_field: source.as
    properties:
    - asn
    - organization_name
    ignore_missing: true
- rename:
    field: source.as.asn
    target_field: source.as.number
    ignore_missing: true
- rename:
    field: source.as.organization_name
    target_field: source.as.organization.name
    ignore_missing: true
- set:
    field: tls.cipher
    value: '{{apache.access.ssl.cipher}}'
    ignore_empty_value: true

- script:
    lang: painless
    if: ctx?.apache?.access?.ssl?.protocol != null
    source: >-
      def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v");
      if (parts.length != 2) {
        return;
      }
      if (parts[1].contains(".")) {
        ctx.tls.version = parts[1];
      } else {
        ctx.tls.version = parts[1] + ".0";
      }
      ctx.tls.version_protocol = parts[0];

- script:
    lang: painless
    description: This script processor iterates over the whole document to remove fields with null values.
    source: |
      void handleMap(Map map) {
        for (def x : map.values()) {
          if (x instanceof Map) {
              handleMap(x);
          } else if (x instanceof List) {
              handleList(x);
          }
        }
        map.values().removeIf(v -> v == null);
      }
      void handleList(List list) {
        for (def x : list) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
        }
      }
      handleMap(ctx);

on_failure:
- set:
    field: error.message
    value: '{{ _ingest.on_failure_message }}'
