Banner

 

rsyslog omelasticsearch bechmark reportΒΆ

/usr/share/elma-tests/bin/elma_elasticsearch_compression_rate.sh
===========================================================================================
Scriptname : elma_elasticsearch_compression_rate.sh
-------------------------------------------------------------------------------------------
Description: rsyslogd performance and elasticsearch compression test
Author     : Joerg Heinemann
Website    : http://enterprise-log-management-appliance.org
===========================================================================================
--- without compression
===========================================================================================
--- 23:47:38.195 - init -------------------------------------------------------------------
--- 23:47:38.228 - delete elasticsearch diag index ----------------------------------------
{"ok":true,"acknowledged":true}
--- 23:47:38.581 - start rsyslogd ---------------------------------------------------------
# author: Joerg Heinemann
# website: http://enterprise-log-management-appliance.org
# license: GPL v3
#
# /opt/elma/config/rsyslog.d/elma_imptcp_elasticsearch-no-compression.conf
#
# simple async writing test to elasticsearch
# Joerg Heinemann, 2012-12-31
$IncludeConfig /opt/elma/config/rsyslog.d/diag-common.conf
$MaxMessageSize 10k
$ModLoad imptcp
$MainMsgQueueTimeoutShutdown 10000
$InputPTCPServerRun 13514
# ######### Enable In-Memory queues ##########
#
# An in-memory queue is created for this action. If the remote host or
# database is down, messages are spooled to memory and sent when it is
# up again.
#
# The advantages are that it will be consistently good in terms of performance
# and it will fail fast.
#
# The downside is that you can keep a limited number of messages in that queue,
# less than you normally can with a disk-assisted queue.
#
# allocate memory dynamically for the queue. Better for handling spikes
$MainMsgQueueType LinkedList
# how many messages (messages, not bytes!) to hold in memory
$MainMsgQueueSize 8000000
# don't throttle receiving messages when the queue gets full
$MainMsgQueueTimeoutEnqueue 0
# ######### Event forwarding to ElasticSearch ##########
#
# Enable ElasticSearch output
#
$ModLoad omelasticsearch.so
# Template for JSON output
#
#$template ls_json,"{\"@timestamp\":\"%timereported:1:19:date-rfc3339%.%timereported:1:3:date-subseconds%\",\"@source_host\":%source:::jsonf%\",\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timereported% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility:facility%,%syslogfacility-text:facility_label%,%syslogseverity:severity%,%syslogseverity-text:severity_label%,%pri:priority%,%app-name:program%,%procid:processid%},\"@customer\":\"AIDB\"}"
$template ls_json,"{%timestamp:::date-rfc3339,jsonf:@timestamp%,%source:::jsonf:@source_host%,\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timestamp% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility:::jsonf:facility%,%syslogfacility-text:::jsonf:facility_label%,%syslogseverity:::jsonf:severity%,%syslogseverity-text:::jsonf:severity_label%,%pri:::jsonf:priority%,%app-name:::jsonf:program%,%procid:::jsonf:processid%},\"@customer\":\"AIDB\",\"@type\":\"events\"}"
$template srchidx,"diag-no-compression"
# now we put everything together
# "template" is for storing the syslog fields we want
#
local4.* action(type="omelasticsearch"
                template="ls_json"
                searchIndex="srchidx"
                dynSearchIndex="on"
                asyncrepl="on"
                searchType="events"
                server="localhost"
                serverport="9200"
#                uid="elma"
#                pwd="fjg892345sdfsdwf43d"
                bulkmode="on"
                queue.dequeuebatchsize="65536"
                queue.type="linkedlist"
                queue.filename="dbq"
                queue.highwatermark="500000"
                queue.lowwatermark="400000"
                queue.discardmark="5000000"
                queue.timeoutenqueue="0"
                queue.maxdiskspace="5g"
                queue.size="2000000"
                queue.saveonshutdown="on"
                action.resumeretrycount="-1")
# There are some other nice things you can use:
#
#    searchIndex               - for letting rsyslog know where to get these names
#    dynSearchIndex="on"       - for having variable index names
#    searchType="mycustomtype" - to specify a different type than "events".
#                                You can have dynSearchType="on" to have it variable, like you can with indices.
#    serverport="9200"         - the default setting, but you can specify a different port
#    asyncrepl="on"            - to enable asyncronous replication.
#                                That is, Elasticsearch gives an answer imediately
#                                after inserting to the main shard(s).
#                                It doesn't wait for replicas to be updated as well, which is the default setting
#    timeout="1m"              - how long to wait for a reply from Elasticsearch.
#                                More info here, near the end:
#                                http://www.elasticsearch.org/guide/reference/api/index_.html
# /opt/elma/config/rsyslog.d/diag-common.conf
#
# This is a config include file. It sets up rsyslog so that the
# diag system can successfully be used. Also, it generates a file
# "rsyslogd.started" after rsyslogd is initialized. This config file
# should be included in all tests that intend to use common code for
# controlling the daemon.
# NOTE: we assume that rsyslogd's current working directory is
# ./tests (or the distcheck equivalent), in particlular that this
# config file resides in the testsuites subdirectory.
# rgerhards, 2009-05-27
$ModLoad imdiag
$IMDiagServerRun 13500
$template startupfile,"rsyslogd.started" # trick to use relative path names!
:syslogtag, contains, "rsyslogd"  ?startupfile
# I have disabled the directive below, so that we see errors in testcase
# creation. I am not sure why it was present in the first place, so for
# now I just leave it commented out -- rgerhards, 2011-03-30
#$ErrorMessagesToStderr off
--- 23:47:38.710 - wait for rsyslogd startup ----------------------------------------------
rsyslogd: error starting up disk queue, using pure in-memory mode [try http://www.rsyslog.com/e/2036 ]
rsyslogd started with pid  15881
--- 23:47:38.832 - run tcpflood -----------------------------------------------------------
00005 open connections
starting run 1
Sending 10000 messages.
00010000 messages sent
runtime: 0.116
sleeping 1 seconds before next run
starting run 2
Sending 10000 messages.
00010000 messages sent
runtime: 0.126
sleeping 1 seconds before next run
starting run 3
Sending 10000 messages.
00010000 messages sent
runtime: 0.114
sleeping 1 seconds before next run
starting run 4
Sending 10000 messages.
00010000 messages sent
runtime: 0.124
sleeping 1 seconds before next run
starting run 5
Sending 10000 messages.
00010000 messages sent
runtime: 0.148
sleeping 1 seconds before next run
starting run 6
Sending 10000 messages.
00010000 messages sent
runtime: 0.127
sleeping 1 seconds before next run
starting run 7
Sending 10000 messages.
00010000 messages sent
runtime: 0.109
sleeping 1 seconds before next run
starting run 8
Sending 10000 messages.
00010000 messages sent
runtime: 0.119
sleeping 1 seconds before next run
starting run 9
Sending 10000 messages.
00010000 messages sent
runtime: 0.077
sleeping 1 seconds before next run
starting run 10
Sending 10000 messages.
00010000 messages sent
runtime: 0.110
Runs:     10
Runtime:
  total:  1.170
  avg:    0.117
  min:    0.077
  max:    0.148
All times are wallclock time.
00005 close connections
End of tcpflood Run
Raw message lenght (Byte):                      230
Messages sent during one tcpflood test:         10000
Number of tcpflood tests:                       10
Seconds to sleep between tcpflood runs:         1
Concurrent tcpflood connections:                5
tcpflood transport protocol:                    tcp
tcpflood rsyslog target port:                   13514
tcpflood rsyslog target address:                127.0.0.1
Total messages:                                 100000
Total tcpflood runtime (milli seconds):         10182
Loging rate (MPS):                              9821
--- 23:47:49.027 - count elasticsearch indexed messages -----------------------------------
37253 messages indexed (1000 milliseconds - Abort in 120000 milliseconds)
74143 messages indexed (2000 milliseconds - Abort in 180000 milliseconds)
74143 messages indexed (3000 milliseconds - Abort in 180000 milliseconds)
74143 messages indexed (4000 milliseconds - Abort in 179000 milliseconds)
74143 messages indexed (5000 milliseconds - Abort in 178000 milliseconds)
74143 messages indexed (6000 milliseconds - Abort in 177000 milliseconds)
100000 messages indexed (7000 milliseconds - Abort in 176000 milliseconds)
--- 23:47:56.280 - show elasticsearch index statistics ------------------------------------
{
  "ok" : true,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "_all" : {
    "primaries" : {
      "docs" : {
        "count" : 100000,
        "deleted" : 0
      },
      "store" : {
        "size" : "77.2mb",
        "size_in_bytes" : 81009144,
        "throttle_time" : "0s",
        "throttle_time_in_millis" : 0
      },
      "indexing" : {
        "index_total" : 100000,
        "index_time" : "12.1s",
        "index_time_in_millis" : 12113,
        "index_current" : 0,
        "delete_total" : 0,
        "delete_time" : "0s",
        "delete_time_in_millis" : 0,
        "delete_current" : 0
      },
      "get" : {
        "total" : 0,
        "time" : "0s",
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time" : "0s",
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time" : "0s",
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "query_total" : 0,
        "query_time" : "0s",
        "query_time_in_millis" : 0,
        "query_current" : 0,
        "fetch_total" : 0,
        "fetch_time" : "0s",
        "fetch_time_in_millis" : 0,
        "fetch_current" : 0
      }
    },
    "total" : {
      "docs" : {
        "count" : 100000,
        "deleted" : 0
      },
      "store" : {
        "size" : "77.2mb",
        "size_in_bytes" : 81009144,
        "throttle_time" : "0s",
        "throttle_time_in_millis" : 0
      },
      "indexing" : {
        "index_total" : 100000,
        "index_time" : "12.1s",
        "index_time_in_millis" : 12113,
        "index_current" : 0,
        "delete_total" : 0,
        "delete_time" : "0s",
        "delete_time_in_millis" : 0,
        "delete_current" : 0
      },
      "get" : {
        "total" : 0,
        "time" : "0s",
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time" : "0s",
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time" : "0s",
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "query_total" : 0,
        "query_time" : "0s",
        "query_time_in_millis" : 0,
        "query_current" : 0,
        "fetch_total" : 0,
        "fetch_time" : "0s",
        "fetch_time_in_millis" : 0,
        "fetch_current" : 0
      }
    },
    "indices" : {
      "diag-no-compression" : {
        "primaries" : {
          "docs" : {
            "count" : 100000,
            "deleted" : 0
          },
          "store" : {
            "size" : "77.2mb",
            "size_in_bytes" : 81009144,
            "throttle_time" : "0s",
            "throttle_time_in_millis" : 0
          },
          "indexing" : {
            "index_total" : 100000,
            "index_time" : "12.1s",
            "index_time_in_millis" : 12113,
            "index_current" : 0,
            "delete_total" : 0,
            "delete_time" : "0s",
            "delete_time_in_millis" : 0,
            "delete_current" : 0
          },
          "get" : {
            "total" : 0,
            "time" : "0s",
            "time_in_millis" : 0,
            "exists_total" : 0,
            "exists_time" : "0s",
            "exists_time_in_millis" : 0,
            "missing_total" : 0,
            "missing_time" : "0s",
            "missing_time_in_millis" : 0,
            "current" : 0
          },
          "search" : {
            "query_total" : 0,
            "query_time" : "0s",
            "query_time_in_millis" : 0,
            "query_current" : 0,
            "fetch_total" : 0,
            "fetch_time" : "0s",
            "fetch_time_in_millis" : 0,
            "fetch_current" : 0
          }
        },
        "total" : {
          "docs" : {
            "count" : 100000,
            "deleted" : 0
          },
          "store" : {
            "size" : "77.2mb",
            "size_in_bytes" : 81009144,
            "throttle_time" : "0s",
            "throttle_time_in_millis" : 0
          },
          "indexing" : {
            "index_total" : 100000,
            "index_time" : "12.1s",
            "index_time_in_millis" : 12113,
            "index_current" : 0,
            "delete_total" : 0,
            "delete_time" : "0s",
            "delete_time_in_millis" : 0,
            "delete_current" : 0
          },
          "get" : {
            "total" : 0,
            "time" : "0s",
            "time_in_millis" : 0,
            "exists_total" : 0,
            "exists_time" : "0s",
            "exists_time_in_millis" : 0,
            "missing_total" : 0,
            "missing_time" : "0s",
            "missing_time_in_millis" : 0,
            "current" : 0
          },
          "search" : {
            "query_total" : 0,
            "query_time" : "0s",
            "query_time_in_millis" : 0,
            "query_current" : 0,
            "fetch_total" : 0,
            "fetch_time" : "0s",
            "fetch_time_in_millis" : 0,
            "fetch_current" : 0
          }
        }
      }
    }
  }
}
--- 23:47:56.314 - show elasticsearch index status ----------------------------------------
{
  "ok" : true,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "indices" : {
    "diag-no-compression" : {
      "index" : {
        "primary_size" : "77.2mb",
        "primary_size_in_bytes" : 81009144,
        "size" : "77.2mb",
        "size_in_bytes" : 81009144
      },
      "translog" : {
        "operations" : 0
      },
      "docs" : {
        "num_docs" : 100000,
        "max_doc" : 100000,
        "deleted_docs" : 0
      },
      "merges" : {
        "current" : 0,
        "current_docs" : 0,
        "current_size" : "0b",
        "current_size_in_bytes" : 0,
        "total" : 0,
        "total_time" : "0s",
        "total_time_in_millis" : 0,
        "total_docs" : 0,
        "total_size" : "0b",
        "total_size_in_bytes" : 0
      },
      "refresh" : {
        "total" : 1,
        "total_time" : "0s",
        "total_time_in_millis" : 0
      },
      "flush" : {
        "total" : 3,
        "total_time" : "1.8s",
        "total_time_in_millis" : 1879
      },
      "shards" : {
        "0" : [ {
          "routing" : {
            "state" : "STARTED",
            "primary" : true,
            "node" : "pqp3jV5fT7Gc2tULGxLoYg",
            "relocating_node" : null,
            "shard" : 0,
            "index" : "diag-no-compression"
          },
          "state" : "STARTED",
          "index" : {
            "size" : "77.2mb",
            "size_in_bytes" : 81009144
          },
          "translog" : {
            "id" : 1363906059093,
            "operations" : 0
          },
          "docs" : {
            "num_docs" : 100000,
            "max_doc" : 100000,
            "deleted_docs" : 0
          },
          "merges" : {
            "current" : 0,
            "current_docs" : 0,
            "current_size" : "0b",
            "current_size_in_bytes" : 0,
            "total" : 0,
            "total_time" : "0s",
            "total_time_in_millis" : 0,
            "total_docs" : 0,
            "total_size" : "0b",
            "total_size_in_bytes" : 0
          },
          "refresh" : {
            "total" : 1,
            "total_time" : "0s",
            "total_time_in_millis" : 0
          },
          "flush" : {
            "total" : 3,
            "total_time" : "1.8s",
            "total_time_in_millis" : 1879
          }
        } ]
      }
    }
  }
}
--- 23:47:56.335 - show elasticsearch index settings --------------------------------------
{
  "diag-no-compression" : {
    "settings" : {
      "index.query.default_field" : "@message",
      "index.refresh_interval" : "30s",
      "index.number_of_replicas" : "0",
      "index.number_of_shards" : "1",
      "index.store.compress.stored" : "false",
      "index.cache.field.type" : "soft",
      "index.version.created" : "200599"
    }
  }
}
--- 23:47:56.422 - show elasticsearch index mappings --------------------------------------
{
  "diag-no-compression" : {
    "events" : {
      "properties" : {
        "@customer" : {
          "type" : "string",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "index_options" : "docs"
        },
        "@fields" : {
          "dynamic" : "true",
          "properties" : {
            "facility" : {
              "type" : "integer"
            },
            "facility_label" : {
              "type" : "string",
              "index" : "not_analyzed",
              "omit_norms" : true,
              "index_options" : "docs"
            },
            "priority" : {
              "type" : "integer"
            },
            "processid" : {
              "type" : "string",
              "index" : "not_analyzed",
              "omit_norms" : true,
              "index_options" : "docs"
            },
            "program" : {
              "type" : "string",
              "index" : "not_analyzed",
              "omit_norms" : true,
              "index_options" : "docs"
            },
            "severity" : {
              "type" : "integer"
            },
            "severity_label" : {
              "type" : "string",
              "index" : "not_analyzed",
              "omit_norms" : true,
              "index_options" : "docs"
            }
          }
        },
        "@message" : {
          "type" : "string"
        },
        "@source" : {
          "type" : "string",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "index_options" : "docs"
        },
        "@source_host" : {
          "type" : "string",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "index_options" : "docs"
        },
        "@timestamp" : {
          "type" : "date",
          "format" : "dateOptionalTime"
        },
        "@type" : {
          "type" : "string",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "index_options" : "docs"
        }
      }
    }
  }
}
--- 23:47:56.451 - shutdown rsyslogd when main queue is empty -----------------------------
--- 23:47:56.474 - wait for main message queue to be empty --------------------------------
imdiag[13500]: mainqueue empty
--- 23:47:57.243 - wait for rsyslogd shutdown ---------------------------------------------
--- 23:47:57.362 - exit -------------------------------------------------------------------
===========================================================================================
PASS:  (without compression)
-------------------------------------------------------------------------------------------
Raw message lenght (Byte):                              230
Total message lenght (Byte):
Messages sent during one tcpflood test:                 10000
Number of tcpflood tests:                               10
Seconds to sleep between tcpflood runs:                 1
Concurrent tcpflood connections:                        5
tcpflood transport protocol:                            tcp
tcpflood rsyslog target port:                           13514
tcpflood rsyslog target address:                        127.0.0.1
Total messages:                                         100000
Total tcpflood runtime (milli seconds):                 10182
Loging rate (MPS):                                      9821
Transmission speed (MBit/s):
Compressed data size (MByte):
Compressed data indexing runtime (milli seconds):
Compression method:                                     index.store.compress.stored = false
Compression ratio (%):
Decompression runtime (milli seconds):
Uncompressed data size (MByte):                         77.25
Uncompressed data indexing runtime (milli seconds):     12113
===========================================================================================
--- with compression
===========================================================================================
--- 23:47:57.397 - init -------------------------------------------------------------------
--- 23:47:57.439 - delete elasticsearch diag index ----------------------------------------
{"ok":true,"acknowledged":true}
--- 23:47:57.486 - start rsyslogd ---------------------------------------------------------
# author: Joerg Heinemann
# website: http://enterprise-log-management-appliance.org
# license: GPL v3
#
# /opt/elma/config/rsyslog.d/elma_imptcp_elasticsearch.conf
#
# simple async writing test to elasticsearch
# Joerg Heinemann, 2012-12-31
$IncludeConfig /opt/elma/config/rsyslog.d/diag-common.conf
$MaxMessageSize 10k
$ModLoad imptcp
$MainMsgQueueTimeoutShutdown 10000
$InputPTCPServerRun 13514
# ######### Enable In-Memory queues ##########
#
# An in-memory queue is created for this action. If the remote host or
# database is down, messages are spooled to memory and sent when it is
# up again.
#
# The advantages are that it will be consistently good in terms of performance
# and it will fail fast.
#
# The downside is that you can keep a limited number of messages in that queue,
# less than you normally can with a disk-assisted queue.
#
# allocate memory dynamically for the queue. Better for handling spikes
$MainMsgQueueType LinkedList
# how many messages (messages, not bytes!) to hold in memory
$MainMsgQueueSize 2000000
# don't throttle receiving messages when the queue gets full
$MainMsgQueueTimeoutEnqueue 0
# ######### Event forwarding to ElasticSearch ##########
#
# Enable ElasticSearch output
#
$ModLoad omelasticsearch.so
# Template for JSON output
#
#$template ls_json,"{\"@timestamp\":\"%timereported:1:19:date-rfc3339%.%timereported:1:3:date-subseconds%\",\"@source_host\":%source:::jsonf%\",\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timereported% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility:facility%,%syslogfacility-text:facility_label%,%syslogseverity:severity%,%syslogseverity-text:severity_label%,%pri:priority%,%app-name:program%,%procid:processid%},\"@customer\":\"AIDB\"}"
$template ls_json,"{%timestamp:::date-rfc3339,jsonf:@timestamp%,%source:::jsonf:@source_host%,\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timestamp% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility:::jsonf:facility%,%syslogfacility-text:::jsonf:facility_label%,%syslogseverity:::jsonf:severity%,%syslogseverity-text:::jsonf:severity_label%,%pri:::jsonf:priority%,%app-name:::jsonf:program%,%procid:::jsonf:processid%},\"@customer\":\"AIDB\",\"@type\":\"events\"}"
$template srchidx,"diag"
# now we put everything together
# "template" is for storing the syslog fields we want
#
local4.* action(type="omelasticsearch"
                template="ls_json"
                searchIndex="srchidx"
                dynSearchIndex="on"
                asyncrepl="on"
                searchType="events"
                server="localhost"
                serverport="9200"
#                uid="elma"
#                pwd="fjg892345sdfsdwf43d"
                bulkmode="on"
                queue.dequeuebatchsize="65536"
                queue.type="linkedlist"
                queue.filename="dbq"
                queue.highwatermark="500000"
                queue.lowwatermark="400000"
                queue.discardmark="2000000"
                queue.timeoutenqueue="0"
                queue.maxdiskspace="5g"
                queue.size="2000000"
                queue.saveonshutdown="on"
                action.resumeretrycount="-1")
# There are some other nice things you can use:
#
#    searchIndex               - for letting rsyslog know where to get these names
#    dynSearchIndex="on"       - for having variable index names
#    searchType="mycustomtype" - to specify a different type than "events".
#                                You can have dynSearchType="on" to have it variable, like you can with indices.
#    serverport="9200"         - the default setting, but you can specify a different port
#    asyncrepl="on"            - to enable asyncronous replication.
#                                That is, Elasticsearch gives an answer imediately
#                                after inserting to the main shard(s).
#                                It doesn't wait for replicas to be updated as well, which is the default setting
#    timeout="1m"              - how long to wait for a reply from Elasticsearch.
#                                More info here, near the end:
#                                http://www.elasticsearch.org/guide/reference/api/index_.html
# /opt/elma/config/rsyslog.d/diag-common.conf
#
# This is a config include file. It sets up rsyslog so that the
# diag system can successfully be used. Also, it generates a file
# "rsyslogd.started" after rsyslogd is initialized. This config file
# should be included in all tests that intend to use common code for
# controlling the daemon.
# NOTE: we assume that rsyslogd's current working directory is
# ./tests (or the distcheck equivalent), in particlular that this
# config file resides in the testsuites subdirectory.
# rgerhards, 2009-05-27
$ModLoad imdiag
$IMDiagServerRun 13500
$template startupfile,"rsyslogd.started" # trick to use relative path names!
:syslogtag, contains, "rsyslogd"  ?startupfile
# I have disabled the directive below, so that we see errors in testcase
# creation. I am not sure why it was present in the first place, so for
# now I just leave it commented out -- rgerhards, 2011-03-30
#$ErrorMessagesToStderr off
--- 23:47:57.608 - wait for rsyslogd startup ----------------------------------------------
rsyslogd: error starting up disk queue, using pure in-memory mode [try http://www.rsyslog.com/e/2036 ]
rsyslogd started with pid  16053
--- 23:47:57.727 - run tcpflood -----------------------------------------------------------
00005 open connections
starting run 1
Sending 10000 messages.
00010000 messages sent
runtime: 0.186
sleeping 1 seconds before next run
starting run 2
Sending 10000 messages.
00010000 messages sent
runtime: 0.088
sleeping 1 seconds before next run
starting run 3
Sending 10000 messages.
00010000 messages sent
runtime: 0.162
sleeping 1 seconds before next run
starting run 4
Sending 10000 messages.
00010000 messages sent
runtime: 0.084
sleeping 1 seconds before next run
starting run 5
Sending 10000 messages.
00010000 messages sent
runtime: 0.131
sleeping 1 seconds before next run
starting run 6
Sending 10000 messages.
00010000 messages sent
runtime: 0.079
sleeping 1 seconds before next run
starting run 7
Sending 10000 messages.
00010000 messages sent
runtime: 0.093
sleeping 1 seconds before next run
starting run 8
Sending 10000 messages.
00010000 messages sent
runtime: 0.123
sleeping 1 seconds before next run
starting run 9
Sending 10000 messages.
00010000 messages sent
runtime: 0.130
sleeping 1 seconds before next run
starting run 10
Sending 10000 messages.
00010000 messages sent
runtime: 0.101
Runs:     10
Runtime:
  total:  1.177
  avg:    0.117
  min:    0.079
  max:    0.186
All times are wallclock time.
00005 close connections
End of tcpflood Run
Raw message lenght (Byte):                      230
Messages sent during one tcpflood test:         10000
Number of tcpflood tests:                       10
Seconds to sleep between tcpflood runs:         1
Concurrent tcpflood connections:                5
tcpflood transport protocol:                    tcp
tcpflood rsyslog target port:                   13514
tcpflood rsyslog target address:                127.0.0.1
Total messages:                                 100000
Total tcpflood runtime (milli seconds):         10190
Loging rate (MPS):                              9813
--- 23:48:07.929 - count elasticsearch indexed messages -----------------------------------
40177 messages indexed (8000 milliseconds - Abort in 120000 milliseconds)
81311 messages indexed (9000 milliseconds - Abort in 180000 milliseconds)
81311 messages indexed (10000 milliseconds - Abort in 180000 milliseconds)
81311 messages indexed (11000 milliseconds - Abort in 179000 milliseconds)
81311 messages indexed (12000 milliseconds - Abort in 178000 milliseconds)
81311 messages indexed (13000 milliseconds - Abort in 177000 milliseconds)
100000 messages indexed (14000 milliseconds - Abort in 176000 milliseconds)
--- 23:48:15.065 - show elasticsearch index statistics ------------------------------------
{
  "ok" : true,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "_all" : {
    "primaries" : {
      "docs" : {
        "count" : 100000,
        "deleted" : 0
      },
      "store" : {
        "size" : "23.4mb",
        "size_in_bytes" : 24585756,
        "throttle_time" : "0s",
        "throttle_time_in_millis" : 0
      },
      "indexing" : {
        "index_total" : 100000,
        "index_time" : "11.3s",
        "index_time_in_millis" : 11353,
        "index_current" : 0,
        "delete_total" : 0,
        "delete_time" : "0s",
        "delete_time_in_millis" : 0,
        "delete_current" : 0
      },
      "get" : {
        "total" : 0,
        "time" : "0s",
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time" : "0s",
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time" : "0s",
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "query_total" : 0,
        "query_time" : "0s",
        "query_time_in_millis" : 0,
        "query_current" : 0,
        "fetch_total" : 0,
        "fetch_time" : "0s",
        "fetch_time_in_millis" : 0,
        "fetch_current" : 0
      }
    },
    "total" : {
      "docs" : {
        "count" : 100000,
        "deleted" : 0
      },
      "store" : {
        "size" : "23.4mb",
        "size_in_bytes" : 24585756,
        "throttle_time" : "0s",
        "throttle_time_in_millis" : 0
      },
      "indexing" : {
        "index_total" : 100000,
        "index_time" : "11.3s",
        "index_time_in_millis" : 11353,
        "index_current" : 0,
        "delete_total" : 0,
        "delete_time" : "0s",
        "delete_time_in_millis" : 0,
        "delete_current" : 0
      },
      "get" : {
        "total" : 0,
        "time" : "0s",
        "time_in_millis" : 0,
        "exists_total" : 0,
        "exists_time" : "0s",
        "exists_time_in_millis" : 0,
        "missing_total" : 0,
        "missing_time" : "0s",
        "missing_time_in_millis" : 0,
        "current" : 0
      },
      "search" : {
        "query_total" : 0,
        "query_time" : "0s",
        "query_time_in_millis" : 0,
        "query_current" : 0,
        "fetch_total" : 0,
        "fetch_time" : "0s",
        "fetch_time_in_millis" : 0,
        "fetch_current" : 0
      }
    },
    "indices" : {
      "diag" : {
        "primaries" : {
          "docs" : {
            "count" : 100000,
            "deleted" : 0
          },
          "store" : {
            "size" : "23.4mb",
            "size_in_bytes" : 24585756,
            "throttle_time" : "0s",
            "throttle_time_in_millis" : 0
          },
          "indexing" : {
            "index_total" : 100000,
            "index_time" : "11.3s",
            "index_time_in_millis" : 11353,
            "index_current" : 0,
            "delete_total" : 0,
            "delete_time" : "0s",
            "delete_time_in_millis" : 0,
            "delete_current" : 0
          },
          "get" : {
            "total" : 0,
            "time" : "0s",
            "time_in_millis" : 0,
            "exists_total" : 0,
            "exists_time" : "0s",
            "exists_time_in_millis" : 0,
            "missing_total" : 0,
            "missing_time" : "0s",
            "missing_time_in_millis" : 0,
            "current" : 0
          },
          "search" : {
            "query_total" : 0,
            "query_time" : "0s",
            "query_time_in_millis" : 0,
            "query_current" : 0,
            "fetch_total" : 0,
            "fetch_time" : "0s",
            "fetch_time_in_millis" : 0,
            "fetch_current" : 0
          }
        },
        "total" : {
          "docs" : {
            "count" : 100000,
            "deleted" : 0
          },
          "store" : {
            "size" : "23.4mb",
            "size_in_bytes" : 24585756,
            "throttle_time" : "0s",
            "throttle_time_in_millis" : 0
          },
          "indexing" : {
            "index_total" : 100000,
            "index_time" : "11.3s",
            "index_time_in_millis" : 11353,
            "index_current" : 0,
            "delete_total" : 0,
            "delete_time" : "0s",
            "delete_time_in_millis" : 0,
            "delete_current" : 0
          },
          "get" : {
            "total" : 0,
            "time" : "0s",
            "time_in_millis" : 0,
            "exists_total" : 0,
            "exists_time" : "0s",
            "exists_time_in_millis" : 0,
            "missing_total" : 0,
            "missing_time" : "0s",
            "missing_time_in_millis" : 0,
            "current" : 0
          },
          "search" : {
            "query_total" : 0,
            "query_time" : "0s",
            "query_time_in_millis" : 0,
            "query_current" : 0,
            "fetch_total" : 0,
            "fetch_time" : "0s",
            "fetch_time_in_millis" : 0,
            "fetch_current" : 0
          }
        }
      }
    }
  }
}
--- 23:48:15.104 - show elasticsearch index status ----------------------------------------
{
  "ok" : true,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "indices" : {
    "diag" : {
      "index" : {
        "primary_size" : "23.4mb",
        "primary_size_in_bytes" : 24585756,
        "size" : "23.4mb",
        "size_in_bytes" : 24585756
      },
      "translog" : {
        "operations" : 0
      },
      "docs" : {
        "num_docs" : 100000,
        "max_doc" : 100000,
        "deleted_docs" : 0
      },
      "merges" : {
        "current" : 0,
        "current_docs" : 0,
        "current_size" : "0b",
        "current_size_in_bytes" : 0,
        "total" : 0,
        "total_time" : "0s",
        "total_time_in_millis" : 0,
        "total_docs" : 0,
        "total_size" : "0b",
        "total_size_in_bytes" : 0
      },
      "refresh" : {
        "total" : 1,
        "total_time" : "0s",
        "total_time_in_millis" : 0
      },
      "flush" : {
        "total" : 3,
        "total_time" : "1.6s",
        "total_time_in_millis" : 1656
      },
      "shards" : {
        "0" : [ {
          "routing" : {
            "state" : "STARTED",
            "primary" : true,
            "node" : "pqp3jV5fT7Gc2tULGxLoYg",
            "relocating_node" : null,
            "shard" : 0,
            "index" : "diag"
          },
          "state" : "STARTED",
          "index" : {
            "size" : "23.4mb",
            "size_in_bytes" : 24585756
          },
          "translog" : {
            "id" : 1363906077910,
            "operations" : 0
          },
          "docs" : {
            "num_docs" : 100000,
            "max_doc" : 100000,
            "deleted_docs" : 0
          },
          "merges" : {
            "current" : 0,
            "current_docs" : 0,
            "current_size" : "0b",
            "current_size_in_bytes" : 0,
            "total" : 0,
            "total_time" : "0s",
            "total_time_in_millis" : 0,
            "total_docs" : 0,
            "total_size" : "0b",
            "total_size_in_bytes" : 0
          },
          "refresh" : {
            "total" : 1,
            "total_time" : "0s",
            "total_time_in_millis" : 0
          },
          "flush" : {
            "total" : 3,
            "total_time" : "1.6s",
            "total_time_in_millis" : 1656
          }
        } ]
      }
    }
  }
}
--- 23:48:15.126 - show elasticsearch index settings --------------------------------------
{
  "diag" : {
    "settings" : {
      "index.query.default_field" : "@message",
      "index.refresh_interval" : "30s",
      "index.number_of_replicas" : "0",
      "index.number_of_shards" : "1",
      "index.store.compress.stored" : "true",
      "index.cache.field.type" : "soft",
      "index.version.created" : "200599"
    }
  }
}
--- 23:48:15.217 - show elasticsearch index mappings --------------------------------------
{
  "diag" : {
    "events" : {
      "properties" : {
        "@customer" : {
          "type" : "string",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "index_options" : "docs"
        },
        "@fields" : {
          "dynamic" : "true",
          "properties" : {
            "facility" : {
              "type" : "integer"
            },
            "facility_label" : {
              "type" : "string",
              "index" : "not_analyzed",
              "omit_norms" : true,
              "index_options" : "docs"
            },
            "priority" : {
              "type" : "integer"
            },
            "processid" : {
              "type" : "string",
              "index" : "not_analyzed",
              "omit_norms" : true,
              "index_options" : "docs"
            },
            "program" : {
              "type" : "string",
              "index" : "not_analyzed",
              "omit_norms" : true,
              "index_options" : "docs"
            },
            "severity" : {
              "type" : "integer"
            },
            "severity_label" : {
              "type" : "string",
              "index" : "not_analyzed",
              "omit_norms" : true,
              "index_options" : "docs"
            }
          }
        },
        "@message" : {
          "type" : "string"
        },
        "@source" : {
          "type" : "string",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "index_options" : "docs"
        },
        "@source_host" : {
          "type" : "string",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "index_options" : "docs"
        },
        "@timestamp" : {
          "type" : "date",
          "format" : "dateOptionalTime"
        },
        "@type" : {
          "type" : "string",
          "index" : "not_analyzed",
          "omit_norms" : true,
          "index_options" : "docs"
        }
      }
    }
  }
}
--- 23:48:15.240 - shutdown rsyslogd when main queue is empty -----------------------------
--- 23:48:15.254 - wait for main message queue to be empty --------------------------------
imdiag[13500]: mainqueue empty
--- 23:48:16.044 - wait for rsyslogd shutdown ---------------------------------------------
--- 23:48:16.059 - exit -------------------------------------------------------------------
===========================================================================================
PASS:   (with compression)
-------------------------------------------------------------------------------------------
Raw message lenght (Byte):                              230
Total message lenght (Byte):
Messages sent during one tcpflood test:                 10000
Number of tcpflood tests:                               10
Seconds to sleep between tcpflood runs:                 1
Concurrent tcpflood connections:                        5
tcpflood transport protocol:                            tcp
tcpflood rsyslog target port:                           13514
tcpflood rsyslog target address:                        127.0.0.1
Total messages:                                         100000
Total tcpflood runtime (milli seconds):                 10190
Loging rate (MPS):                                      9813
Transmission speed (MBit/s):
Compressed data size (MByte):                           23.44
Compressed data indexing runtime (milli seconds):       11353
Compression method:                                     index.store.compress.stored = true
Compression ratio (%):                                  69.65
Decompression runtime (milli seconds):
Uncompressed data size (MByte):                         77.25
Uncompressed data indexing runtime (milli seconds):     12113
===========================================================================================