Commit ccd20603 authored by Arturo Borrero Gonzalez's avatar Arturo Borrero Gonzalez 🏕
Browse files

New upstream version 0.1.4

parent 6f471e17
[flake8]
exclude = .venv,src/vendor
max-complexity = 10
max-line-length = 120
*.pyc
*.snap
*.swp
*~
.venv
/bin
/pkg
__pycache__
debian/files
parts
prime
*.snap
snap
stage
FROM python:2.7-alpine
RUN \
apk update && \
apk --no-cache -q add build-base linux-headers libffi-dev openssl-dev python2-dev && \
apk --no-cache -q add git gcc make autoconf automake libtool libxml2-dev libxslt-dev && \
cd /tmp && git clone https://github.com/openstack/liberasurecode && \
cd liberasurecode && ./autogen.sh && ./configure && make && make test && make install && \
cd / && rm -rf /tmp/liberasurecode && \
pip install -U pip && \
pip install python-neutronclient python-novaclient python-keystoneclient python-cinderclient \
prometheus-client requests pyyaml netaddr swift flake8
COPY prometheus-openstack-exporter /
COPY prometheus-openstack-exporter.sample.yaml /
COPY wrapper.sh /
EXPOSE 9183
ENTRYPOINT ["/bin/sh", "/wrapper.sh"]
PROJECTPATH = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
VENV := .venv
VENV_PIP := $(PROJECTPATH)/$(VENV)/bin/pip
VENV_PYTHON := $(PROJECTPATH)/$(VENV)/bin/python2.7
EXTRA_PY := $(PROJECTPATH)/prometheus-openstack-exporter
FLAKE8 := $(VENV_PYTHON) -m flake8
all: lint test build
clean: clean-python clean-venv
clean-python:
rm -rf $(PROJECTPATH)/__pycache__
clean-venv:
rm -rf $(PROJECTPATH)/$(VENV)
dch:
gbp dch --debian-tag='%(version)s' -D bionic --git-log --first-parent
deb-src:
debuild -S -sa -I.git
install-build-depends:
sudo apt install \
debhelper \
git-buildpackage
lint: lint-python
# See .flake8 for config options.
lint-python: $(VENV)
$(FLAKE8) $(PROJECTPATH) $(EXTRA_PY)
test: test-python
test-python: $(VENV)
$(VENV_PYTHON) -m unittest discover tests
$(VENV):
virtualenv --system-site-packages -p python2.7 $(PROJECTPATH)/$(VENV)
$(VENV_PIP) install -I -r requirements.txt
......@@ -2,6 +2,7 @@
Exposes high level [OpenStack](http://www.openstack.org/) metrics to [Prometheus](https://prometheus.io/).
Data can be visualised using [Grafana](https://grafana.com/) and the [OpenStack Clouds Dashboard](https://grafana.com/dashboards/7924)
# Deployment
......@@ -63,6 +64,34 @@ Or to run interactively:
```
Or use Docker Image:
```
# docker-compose.yml
version: '2.1'
services:
ostackexporter:
image: moghaddas/prom-openstack-exporter:latest
# check this examle env file
env_file:
- ./admin.novarc.example
restart: unless-stopped
expose:
- 9183
ports:
- 9183:9183
# docker run
docker run \
-itd \
--name prom_openstack_exporter \
-p 9183:9183 \
--env-file=$(pwd)/admin.novarc.example \
--restart=unless-stopped \
moghaddas/prom-openstack-exporter:latest
```
# Configuration
Configuration options are documented in prometheus-openstack-exporter.yaml shipped with this project
......@@ -87,3 +116,25 @@ Swift stats are included mainly because they are trivial to retrieve. If and whe
We are aware that Prometheus best practise is to avoid caching. Unfortunately queries we need to run are very heavy and in bigger clouds can take minutes to execute. This is problematic not only because of delays but also because multiple servers scraping the exporter could have negative impact on the cloud performance
## How are Swift account metrics obtained?
Fairly simply! Given a copy of the Swift rings (in fact, we just need
account.ring.gz) we can load this up and then ask it where particular
accounts are located in the cluster. We assume that Swift is
replicating properly, pick a node at random, and ask it for the
account's statistics with an HTTP HEAD request, which it returns.
## How hard would it be to export Swift usage by container?
Sending a GET request to the account URL yields a list of containers
(probably paginated, so watch out for that!). In order to write a
container-exporter, one could add some code to fetch a list of
containers from the account server, load up the container ring, and
then use container_ring.get_nodes(account, container) and HTTP HEAD on
one of the resulting nodes to get a containers' statistics, although
without some caching cleverness this will scale poorly.
# Known Issues
## EOFError by pickle.py
You should wait. It needs dump file to generate metrics
OS_PROJECT_DOMAIN_NAME=Default
OS_USER_DOMAIN_NAME=Default
OS_PROJECT_NAME=admin
OS_TENANT_NAME=admin
OS_USERNAME=admin
OS_PASSWORD=XXXXXXXXXX
OS_AUTH_URL=http://XX.XX.XX.XX:35357/v3
OS_INTERFACE=internal
OS_IDENTITY_API_VERSION=3
OS_REGION_NAME=mycloud
#listenPort=9183
#cacheRefreshInterval=300
#vcpuRatio=1.0
#ramRatio=1.0
#diskRatio=1.0
#enabledCollectors=cinder neutron nova
#schedulableInstanceRam=4096
#schedulableInstanceVcpu=2
#schedulableInstanceDisk=20
#useNovaVolumes=True
#swiftHosts=host1.example.com host2.example.com
#keystoneTenantsMap="firstname,1234567890 secondname,0987654321"
#resellerPrefix=AUTH_
#ringPath=/etc/swift
#hashPathPrefix=
#hashPathSuffix=
This diff is collapsed.
# Example configuration file for prometheus-openstack-exporter
# Copyright (C) 2016-2019 Canonical, Ltd.
#
listen_port: VAR_LISTEN_PORT # listenPort=9183
cache_refresh_interval: VAR_CACHE_REFRESH_INTERVAL # cacheRefreshInterval=300 In seconds
cache_file: VAR_CACHE_FILE # cacheFileName=$(mktemp -p /dev/shm/)
cloud: VAR_CLOUD # cloud=${OS_REGION_NAME:-mycloud}
openstack_allocation_ratio_vcpu: VAR_VCPU_RATIO # vcpuRatio=1.0
openstack_allocation_ratio_ram: VAR_RAM_RATIO # ramRatio=1.0
openstack_allocation_ratio_disk: VAR_DISK_RATIO # diskRatio=1.0
# Configure the enabled collectors here. Note that the Swift account
# collector in particular has special requirements.
enabled_collectors: # enabledCollectors=cinder neutron nova
- VAR_ENABLED_COLLECTORS
# To export hypervisor_schedulable_instances metric set desired instance size
schedulable_instance_size:
ram_mbs: VAR_SCHEDULABLE_INSTANCE_RAM # schedulableInstanceRam=4096
vcpu: VAR_SCHEDULABLE_INSTANCE_VCPU # schedulableInstanceVcpu=2
disk_gbs: VAR_SCHEDULABLE_INSTANCE_DISK # schedulableInstanceDisk=20
# Uncomment if the cloud doesn't provide cinder / nova volumes:
use_nova_volumes: VAR_USE_NOVA_VOLUMES # useNovaVolumes=True
## Swift
# There is no way to retrieve them using OpenStack APIs
# For clouds deployed without swift, remove this part
swift_hosts: # swiftHosts=host1.example.com host2.example.com
- VAR_SWIFT_HOSTS
# There is no API to ask Swift for a list of accounts it knows about.
# Even if there were, Swift (in common case of Keystone auth, at
# least) only knows them by the corresponding tenant ID, which would
# be a less than useful label without post-processing. The following
# should point to a file containing one line per tenant, with the
# tenant name first, then whitespace, followed by the tenant ID.
keystone_tenants_map: # keystoneTenantsMap="firstname,1234567890 secondname,0987654321"
- VAR_KEYSTONE_TENANTS_MAP
# The reseller prefix is typically used by the Swift middleware to
# keep accounts with different providers separate. We would ideally
# look this up dynamically from the Swift configuration.
# The Keystone middlware defaults to the following value.
reseller_prefix: VAR_RESELLER_PREFIX # resellerPrefix=AUTH_
ring_path: VAR_RING_PATH # ringPath=/etc/swift
# These will typically be read from /etc/swift/swift.conf. If that
# file cannot be opened, then the Swift library will log an error and
# try to exit. To run p-s-a-e as a user other than Swift, these
# settings must be set to the same values as Swift itself, and the
# above must point to an always-current readable copy of the rings.
hash_path_prefix: VAR_HASH_PATH_PREFIX # hashPathPrefix=
hash_path_suffix: VAR_HASH_PATH_SUFFIX # hashPathSuffix=
# Example configuration file for prometheus-openstack-exporter
# Copyright (C) 2016 Canonical, Ltd.
# Copyright (C) 2016-2019 Canonical, Ltd.
#
listen_port: 9183
......@@ -9,6 +9,16 @@ cloud: mycloud
openstack_allocation_ratio_vcpu: 2.5
openstack_allocation_ratio_ram: 1.1
openstack_allocation_ratio_disk: 1.0
log_level: INFO
# Configure the enabled collectors here. Note that the Swift account
# collector in particular has special requirements.
enabled_collectors:
- cinder
- neutron
- nova
- swift
# - swift-account-usage
# To export hypervisor_schedulable_instances metric set desired instance size
schedulable_instance_size:
......@@ -16,11 +26,38 @@ schedulable_instance_size:
vcpu: 2
disk_gbs: 20
# Uncomment if the cloud doesn't provide cinder / nova volumes:
#use_nova_volumes: False
## Swift
# There is no way to retrieve them using OpenStack APIs
# For clouds deployed without swift, remove this part
swift_hosts:
- host1.example.com
- host2.example.com
# Uncomment if the cloud doesn't provide cinder / nova volumes:
#use_nova_volumes: False
# There is no API to ask Swift for a list of accounts it knows about.
# Even if there were, Swift (in common case of Keystone auth, at
# least) only knows them by the corresponding tenant ID, which would
# be a less than useful label without post-processing. The following
# should point to a file containing one line per tenant, with the
# tenant name first, then whitespace, followed by the tenant ID.
keystone_tenants_map:
# The reseller prefix is typically used by the Swift middleware to
# keep accounts with different providers separate. We would ideally
# look this up dynamically from the Swift configuration.
# The Keystone middlware defaults to the following value.
reseller_prefix: AUTH_
ring_path: /etc/swift
# These will typically be read from /etc/swift/swift.conf. If that
# file cannot be opened, then the Swift library will log an error and
# try to exit. To run p-s-a-e as a user other than Swift, these
# settings must be set to the same values as Swift itself, and the
# above must point to an always-current readable copy of the rings.
hash_path_prefix:
hash_path_suffix:
prometheus-openstack-exporter
\ No newline at end of file
......@@ -5,6 +5,7 @@ from setuptools import setup
def read(fname):
return open(os.path.join(os.path.dirname(__file__), fname)).read()
setup(
name="prometheus_openstack_exporter",
version="0.0.4",
......@@ -14,17 +15,20 @@ setup(
keywords=["prometheus", "openstack", "exporter"],
url="https://github.com/CanonicalLtd/prometheus-openstack-exporter",
scripts=["prometheus-openstack-exporter"],
install_requires=["prometheus_client",
"python-keystoneclient<=3.10.0",
"python-novaclient==6.0.0",
"python-neutronclient<=6.1.0",
"python-cinderclient",
"netaddr"],
install_requires=[
"prometheus_client",
"python-keystoneclient<=3.10.0",
"python-novaclient==6.0.0",
"python-neutronclient<=6.1.0",
"python-cinderclient",
"netaddr",
"swift",
],
long_description=read('README.md'),
classifiers=[
"Development Status :: 4 - Beta",
"Topic :: System :: Networking :: Monitoring",
"License :: OSI Approved :: "
"GNU General Public License v3 or later (GPLv3+)",
"GNU General Public License v3 or later (GPLv3+)",
],
)
!GlobalState
assets:
build-packages: []
build-snaps: []
name: prometheus-openstack-exporter
version: 0.0.8
base: core
version: git
summary: Exposes high level OpenStack metrics to Prometheus.
description: |
Exposes high level OpenStack metrics to Prometheus
......@@ -18,6 +19,7 @@ parts:
source: .
stage-packages:
- libdb5.3
- liberasurecode-dev
example-config:
plugin: dump
source: .
......
import unittest
from mock import Mock, call, patch
from requests.structures import CaseInsensitiveDict
import prometheus_openstack_exporter as poe
class TestSwiftAccountUsage(unittest.TestCase):
@patch('prometheus_openstack_exporter.SwiftAccountUsage._get_account_ring')
@patch('prometheus_openstack_exporter.requests.head')
@patch('prometheus_openstack_exporter.config')
def test__get_account_usage(self, _config, _requests_head, _SwiftAccountUsage__get_account_ring):
s = poe.SwiftAccountUsage()
s.account_ring.get_nodes.return_value = (26701, [
{'device': 'sdb', 'id': 0, 'ip': '10.24.0.18', 'meta': u'', 'port': 6002,
'region': 1, 'replication_ip': '10.24.0.18', 'replication_port': 6002, 'weight': 100.0, 'zone': 1},
{'device': 'sdd', 'id': 50, 'ip': '10.24.0.71', 'meta': u'', 'port': 6002,
'region': 1, 'replication_ip': '10.24.0.71', 'replication_port': 6002, 'weight': 180.0, 'zone': 3},
{'device': 'sdi', 'id': 59, 'ip': '10.24.0.72', 'meta': u'', 'port': 6002,
'region': 1, 'replication_ip': '10.24.0.72', 'replication_port': 6002, 'weight': 360.0, 'zone': 2}
])
response_mock = Mock()
response_mock.configure_mock(
status_code=204,
headers=CaseInsensitiveDict({'x-account-bytes-used': '368259416'}),
)
_requests_head.return_value = response_mock
# Assert that _get_account_ring does what we expect.
self.assertEqual(s._get_account_usage('AUTH_12bb569bf909441b90791482ae6f9ca9'), 368259416)
# Assert that _get_account_ring did it in the manner we expected.
s.account_ring.get_nodes.assert_called_once_with(account='AUTH_12bb569bf909441b90791482ae6f9ca9')
poe.requests.head.assert_called_once()
self.assertTrue(poe.requests.head.call_args in [
call('http://10.24.0.18:6002/sdb/26701/AUTH_12bb569bf909441b90791482ae6f9ca9'),
call('http://10.24.0.71:6002/sdd/26701/AUTH_12bb569bf909441b90791482ae6f9ca9'),
call('http://10.24.0.72:6002/sdi/26701/AUTH_12bb569bf909441b90791482ae6f9ca9'),
])
import unittest
import mock
import prometheus_openstack_exporter as poe
class TestPrometheusOpenstackExporter(unittest.TestCase):
def test_data_gatherer_needed(self):
self.assertTrue(
poe.data_gatherer_needed(
{'enabled_collectors':
['cinder', 'neutron', 'nova', 'swift']}))
self.assertTrue(
poe.data_gatherer_needed(
{'enabled_collectors':
['cinder', 'neutron', 'nova', 'swift', 'swift-account-usage']}))
self.assertFalse(
poe.data_gatherer_needed(
{'enabled_collectors':
['swift-account-usage']}))
self.assertFalse(
poe.data_gatherer_needed(
{'enabled_collectors':
['swift', 'swift-account-usage']}))
self.assertEqual(
poe.data_gatherer_needed(
{'enabled_collectors':
['cinder', 'neutron', 'nova', 'swift']}),
set(['cinder', 'nova', 'neutron']),
)
self.assertEqual(
poe.data_gatherer_needed({}),
set(['cinder', 'nova', 'neutron']),
)
@mock.patch('prometheus_openstack_exporter.config')
def test_get_nova_info(self, config):
config.return_value = {}
prodstack = {'tenants': []}
nova = mock.Mock()
nova.aggregates = mock.MagicMock()
nova.flavors = mock.MagicMock()
nova.hypervisors = mock.MagicMock()
nova.servers = mock.MagicMock()
nova.servers.list = mock.MagicMock()
nova.services = mock.MagicMock()
data_gatherer = poe.DataGatherer()
data_gatherer._get_nova_info(nova, None, prodstack)
expected = [
mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'ACTIVE', 'all_tenants': '1'}),
mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'ERROR', 'all_tenants': '1'}),
mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'SHELVED_OFFLOADED', 'all_tenants': '1'}),
mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'SHUTOFF', 'all_tenants': '1'}),
mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'SUSPENDED', 'all_tenants': '1'}),
mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'VERIFY_RESIZE', 'all_tenants': '1'}),
]
nova.servers.list.assert_has_calls(expected, any_order=True)
not_expected = mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'BUILD', 'all_tenants': '1'})
self.assertTrue(not_expected not in nova.servers.list.call_args_list)
#!/bin/sh
prometheusDir='/etc/prometheus'
configFile=${configFile:-"${prometheusDir}/prometheus-openstack-exporter.yaml"}
listenPort=${listenPort:-9183}
cacheRefreshInterval=${cacheRefreshInterval:-300}
cacheFileName=${cacheFileName:-"$(mktemp -p /dev/shm/)"}
cloud=${OS_REGION_NAME:-mycloud}
vcpuRatio=${vcpuRatio:-1.0}
ramRatio=${ramRatio:-1.0}
diskRatio=${diskRatio:-1.0}
enabledCollectors=${enabledCollectors:-cinder neutron nova}
schedulableInstanceRam=${schedulableInstanceRam:-4096}
schedulableInstanceVcpu=${schedulableInstanceVcpu:-2}
schedulableInstanceDisk=${schedulableInstanceDisk:-20}
useNovaVolumes=${useNovaVolumes:-True}
swiftHosts=${swiftHosts:-host1.example.com host2.example.com}
#keystoneTenantsMap="firstname,1234567890 secondname,0987654321"
resellerPrefix=${resellerPrefix:-AUTH_}
ringPath=${ringPath:-/etc/swift}
#hashPathPrefix=
#hashPathSuffix=
if [ ! -e "${configFile}" ]; then
mkdir -p ${prometheusDir}
cp prometheus-openstack-exporter.sample.yaml ${configFile}
sed -i "s|VAR_LISTEN_PORT|${listenPort}|g" ${configFile}
sed -i "s|VAR_CACHE_REFRESH_INTERVAL|${cacheRefreshInterval}|g" ${configFile}
sed -i "s|VAR_CACHE_FILE|${cacheFileName}|g" ${configFile}
sed -i "s|VAR_CLOUD|${cloud}|g" ${configFile}
sed -i "s|VAR_VCPU_RATIO|${vcpuRatio}|g" ${configFile}
sed -i "s|VAR_RAM_RATIO|${ramRatio}|g" ${configFile}
sed -i "s|VAR_DISK_RATIO|${diskRatio}|g" ${configFile}
sed -i "s|VAR_SCHEDULABLE_INSTANCE_RAM|${schedulableInstanceRam}|g" ${configFile}
sed -i "s|VAR_SCHEDULABLE_INSTANCE_VCPU|${schedulableInstanceVcpu}|g" ${configFile}
sed -i "s|VAR_SCHEDULABLE_INSTANCE_DISK|${schedulableInstanceDisk}|g" ${configFile}
sed -i "s|VAR_USE_NOVA_VOLUMES|${useNovaVolumes}|g" ${configFile}
for i in ${enabledCollectors}; do
sed -i "s/.*VAR_ENABLED_COLLECTORS/ - ${i}\n - VAR_ENABLED_COLLECTORS/g" ${configFile}
done
sed -i '/.*VAR_ENABLED_COLLECTORS.*/d' ${configFile}
for i in ${swiftHosts}; do
sed -i "s/.*VAR_SWIFT_HOSTS/ - ${i}\n - VAR_SWIFT_HOSTS/g" ${configFile}
done
sed -i '/.*VAR_SWIFT_HOSTS.*/d' ${configFile}
for i in ${keystoneTenantsMap}; do
tenantName=$(echo ${i} | cut -d',' -f1)
tenantId=$( echo ${i} | cut -d',' -f2)
sed -i "s/.*VAR_KEYSTONE_TENANTS_MAP/ - ${tenantName} ${tenantId}\n - VAR_KEYSTONE_TENANTS_MAP/g" ${configFile}
done
sed -i '/.*VAR_KEYSTONE_TENANTS_MAP.*/d' ${configFile}
sed -i "s|VAR_RESELLER_PREFIX|${resellerPrefix}|g" ${configFile}
sed -i "s|VAR_RING_PATH|${ringPath}|g" ${configFile}
sed -i "s|VAR_HASH_PATH_PREFIX|${hashPathPrefix}|g" ${configFile}
sed -i "s|VAR_HASH_PATH_SUFFIX|${hashPathSuffix}|g" ${configFile}
sed -i 's/VAR_.*//g' ${configFile}
touch ${cacheFileName}
cat ${configFile}
fi
/prometheus-openstack-exporter ${configFile}
rm ${cacheFileName}
exit 0
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment