From db0492456d4036ec8155c33202bbe43e99f0ffcf Mon Sep 17 00:00:00 2001 From: David Stapleton Date: Wed, 28 Feb 2018 20:26:51 +0000 Subject: [PATCH 1/2] Add new 'stagger_collection' option The stagger functionality was added in 87da1c5. This works, however in cases where only one collector is configured, it's possible that the maximum collection time calculated is not long enough and the collection is terminated with, "Took too long to run!". This is despite the fact that the configured interval gives more than enough time for the collector to finish its run. Therefore, add a new 'stagger_collection' option that stops this behaviour. When disabled/cleared/set to false, the maximum collection time will remain static, or in other words it will be set to the interval configured and will not be increased if the collector doesn't finish within this time period. This change could potentially screw up existing deployments that rely on the existing stagger functionality, so by default 'stagger_collection' is enabled. --- src/diamond/collector.py | 3 +++ src/diamond/utils/scheduler.py | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/diamond/collector.py b/src/diamond/collector.py index ca8d6f649..29a6dc5f9 100644 --- a/src/diamond/collector.py +++ b/src/diamond/collector.py @@ -291,6 +291,9 @@ def get_default_config(self): # Default Poll Interval (seconds) 'interval': 300, + # Staggering collector runs is enabled by default + 'stagger_collection': True, + # Default Event TTL (interval multiplier) 'ttl_multiplier': 2, diff --git a/src/diamond/utils/scheduler.py b/src/diamond/utils/scheduler.py index ffd1bf856..0ce8da6e9 100644 --- a/src/diamond/utils/scheduler.py +++ b/src/diamond/utils/scheduler.py @@ -13,6 +13,7 @@ except ImportError: setproctitle = None +from diamond.collector import str_to_bool from diamond.utils.signals import signal_to_exception from diamond.utils.signals import SIGALRMException from diamond.utils.signals import SIGHUPException @@ -36,6 +37,7 @@ def collector_process(collector, metric_queue, log): interval = float(collector.config['interval']) log.debug('Starting') + log.debug('Interval: %s seconds', interval) # Validate the interval @@ -43,10 +45,16 @@ def collector_process(collector, metric_queue, log): log.critical('interval of %s is not valid!', interval) sys.exit(1) + # Initially set stagger_offset to 0. If stagger_collection is enabled in + # the config, this value will soon be overridden. + stagger_offset = 0 + # Start the next execution at the next window plus some stagger delay to # avoid having all collectors running at the same time next_window = math.floor(time.time() / interval) * interval - stagger_offset = random.uniform(0, interval - 1) + + if str_to_bool(self.config['stagger_collection']): + stagger_offset = random.uniform(0, interval - 1) # Allocate time till the end of the window for the collector to run. With a # minimum of 1 second From 2c75ab27bf2fb66daf44393356ff32d54c518812 Mon Sep 17 00:00:00 2001 From: David Stapleton Date: Wed, 28 Feb 2018 21:08:30 +0000 Subject: [PATCH 2/2] Minor scheduler log improvements --- src/diamond/utils/scheduler.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/diamond/utils/scheduler.py b/src/diamond/utils/scheduler.py index 0ce8da6e9..7e14d53fc 100644 --- a/src/diamond/utils/scheduler.py +++ b/src/diamond/utils/scheduler.py @@ -36,15 +36,14 @@ def collector_process(collector, metric_queue, log): interval = float(collector.config['interval']) - log.debug('Starting') - - log.debug('Interval: %s seconds', interval) - # Validate the interval if interval <= 0: - log.critical('interval of %s is not valid!', interval) + log.critical('Interval of %s is not valid!', interval) sys.exit(1) + log.debug('Starting %s', collector.name) + log.debug('Interval: %s seconds', interval) + # Initially set stagger_offset to 0. If stagger_collection is enabled in # the config, this value will soon be overridden. stagger_offset = 0 @@ -103,7 +102,7 @@ def collector_process(collector, metric_queue, log): # us and end up with half a loaded config signal.alarm(0) - log.info('Reloading config reload due to HUP') + log.info('Reloading config due to SIGHUP') collector.load_config() log.info('Config reloaded')