New upstream version 0.1.4

.flake8

+[flake8]
+exclude = .venv,src/vendor
+max-complexity = 10
+max-line-length = 120

.gitignore

+*.pyc
+*.snap
+*.swp
+*~
+.venv
+/bin
+/pkg
+__pycache__
+debian/files
 parts
 prime
-*.snap
-snap
 stage

Dockerfile

+FROM python:2.7-alpine
+
+RUN \
+    apk update && \
+    apk --no-cache -q add build-base linux-headers libffi-dev openssl-dev python2-dev && \
+    apk --no-cache -q add git gcc make autoconf automake libtool libxml2-dev libxslt-dev && \
+    cd /tmp && git clone https://github.com/openstack/liberasurecode && \
+    cd liberasurecode && ./autogen.sh && ./configure && make && make test && make install && \
+    cd / && rm -rf /tmp/liberasurecode && \
+    pip install -U pip && \
+    pip install python-neutronclient python-novaclient python-keystoneclient python-cinderclient \
+    prometheus-client requests pyyaml netaddr swift flake8
+
+COPY prometheus-openstack-exporter /
+COPY prometheus-openstack-exporter.sample.yaml /
+COPY wrapper.sh /
+
+EXPOSE 9183
+ENTRYPOINT ["/bin/sh", "/wrapper.sh"]

Makefile

+PROJECTPATH = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+
+VENV := .venv
+VENV_PIP := $(PROJECTPATH)/$(VENV)/bin/pip
+VENV_PYTHON := $(PROJECTPATH)/$(VENV)/bin/python2.7
+
+EXTRA_PY := $(PROJECTPATH)/prometheus-openstack-exporter
+
+FLAKE8 := $(VENV_PYTHON) -m flake8
+
+all: lint test build
+
+clean: clean-python clean-venv
+
+clean-python:
+	rm -rf $(PROJECTPATH)/__pycache__
+
+clean-venv:
+	rm -rf $(PROJECTPATH)/$(VENV)
+
+dch:
+	gbp dch --debian-tag='%(version)s' -D bionic --git-log --first-parent
+
+deb-src:
+	debuild -S -sa -I.git
+
+install-build-depends:
+	sudo apt install \
+	    debhelper \
+	    git-buildpackage
+
+lint: lint-python
+
+# See .flake8 for config options.
+lint-python: $(VENV)
+	$(FLAKE8) $(PROJECTPATH) $(EXTRA_PY)
+
+test: test-python
+
+test-python: $(VENV)
+	$(VENV_PYTHON) -m unittest discover tests
+
+$(VENV):
+	virtualenv --system-site-packages -p python2.7 $(PROJECTPATH)/$(VENV)
+	$(VENV_PIP) install -I -r requirements.txt

README.md

@@ -2,6 +2,7 @@
 
 Exposes high level [OpenStack](http://www.openstack.org/) metrics to [Prometheus](https://prometheus.io/).
 
+Data can be visualised using [Grafana](https://grafana.com/) and the [OpenStack Clouds Dashboard](https://grafana.com/dashboards/7924)
 
 # Deployment
 
@@ -63,6 +64,34 @@ Or to run interactively:
 
 ```
 
+Or use Docker Image:
+
+```
+# docker-compose.yml
+version: '2.1'
+services:
+  ostackexporter:
+    image: moghaddas/prom-openstack-exporter:latest
+    # check this examle env file
+    env_file:
+      - ./admin.novarc.example
+    restart: unless-stopped
+    expose:
+      - 9183
+    ports:
+      - 9183:9183
+
+# docker run
+docker run \
+  -itd \
+  --name prom_openstack_exporter \
+  -p 9183:9183 \
+  --env-file=$(pwd)/admin.novarc.example \
+  --restart=unless-stopped \
+  moghaddas/prom-openstack-exporter:latest
+
+```
+
 # Configuration
 
 Configuration options are documented in prometheus-openstack-exporter.yaml shipped with this project
@@ -87,3 +116,25 @@ Swift stats are included mainly because they are trivial to retrieve. If and whe
 
 We are aware that Prometheus best practise is to avoid caching. Unfortunately queries we need to run are very heavy and in bigger clouds can take minutes to execute. This is problematic not only because of delays but also because multiple servers scraping the exporter could have negative impact on the cloud performance
 
+## How are Swift account metrics obtained?
+
+Fairly simply!  Given a copy of the Swift rings (in fact, we just need
+account.ring.gz) we can load this up and then ask it where particular
+accounts are located in the cluster.  We assume that Swift is
+replicating properly, pick a node at random, and ask it for the
+account's statistics with an HTTP HEAD request, which it returns.
+
+## How hard would it be to export Swift usage by container?
+
+Sending a GET request to the account URL yields a list of containers
+(probably paginated, so watch out for that!).  In order to write a
+container-exporter, one could add some code to fetch a list of
+containers from the account server, load up the container ring, and
+then use container_ring.get_nodes(account, container) and HTTP HEAD on
+one of the resulting nodes to get a containers' statistics, although
+without some caching cleverness this will scale poorly.
+
+# Known Issues
+## EOFError by pickle.py
+
+You should wait. It needs dump file to generate metrics

admin.novarc.example

+OS_PROJECT_DOMAIN_NAME=Default
+OS_USER_DOMAIN_NAME=Default
+OS_PROJECT_NAME=admin
+OS_TENANT_NAME=admin
+OS_USERNAME=admin
+OS_PASSWORD=XXXXXXXXXX
+OS_AUTH_URL=http://XX.XX.XX.XX:35357/v3
+OS_INTERFACE=internal
+OS_IDENTITY_API_VERSION=3
+OS_REGION_NAME=mycloud
+
+#listenPort=9183
+#cacheRefreshInterval=300
+#vcpuRatio=1.0
+#ramRatio=1.0
+#diskRatio=1.0
+#enabledCollectors=cinder neutron nova
+#schedulableInstanceRam=4096
+#schedulableInstanceVcpu=2
+#schedulableInstanceDisk=20
+#useNovaVolumes=True
+#swiftHosts=host1.example.com host2.example.com
+#keystoneTenantsMap="firstname,1234567890 secondname,0987654321"
+#resellerPrefix=AUTH_
+#ringPath=/etc/swift
+#hashPathPrefix=
+#hashPathSuffix=

prometheus-openstack-exporter.sample.yaml

+# Example configuration file for prometheus-openstack-exporter
+# Copyright (C) 2016-2019 Canonical, Ltd.
+#
+
+listen_port: VAR_LISTEN_PORT # listenPort=9183
+cache_refresh_interval: VAR_CACHE_REFRESH_INTERVAL # cacheRefreshInterval=300 In seconds
+cache_file: VAR_CACHE_FILE # cacheFileName=$(mktemp -p /dev/shm/)
+cloud: VAR_CLOUD # cloud=${OS_REGION_NAME:-mycloud}
+openstack_allocation_ratio_vcpu: VAR_VCPU_RATIO # vcpuRatio=1.0
+openstack_allocation_ratio_ram: VAR_RAM_RATIO # ramRatio=1.0
+openstack_allocation_ratio_disk: VAR_DISK_RATIO # diskRatio=1.0
+
+# Configure the enabled collectors here.  Note that the Swift account
+# collector in particular has special requirements.
+enabled_collectors: # enabledCollectors=cinder neutron nova
+  - VAR_ENABLED_COLLECTORS
+
+# To export hypervisor_schedulable_instances metric set desired instance size
+schedulable_instance_size:
+    ram_mbs: VAR_SCHEDULABLE_INSTANCE_RAM # schedulableInstanceRam=4096
+    vcpu: VAR_SCHEDULABLE_INSTANCE_VCPU # schedulableInstanceVcpu=2
+    disk_gbs: VAR_SCHEDULABLE_INSTANCE_DISK # schedulableInstanceDisk=20
+
+# Uncomment if the cloud doesn't provide cinder / nova volumes:
+use_nova_volumes: VAR_USE_NOVA_VOLUMES # useNovaVolumes=True
+
+## Swift
+
+# There is no way to retrieve them using OpenStack APIs
+# For clouds deployed without swift, remove this part
+swift_hosts: # swiftHosts=host1.example.com host2.example.com
+    - VAR_SWIFT_HOSTS
+
+# There is no API to ask Swift for a list of accounts it knows about.
+# Even if there were, Swift (in common case of Keystone auth, at
+# least) only knows them by the corresponding tenant ID, which would
+# be a less than useful label without post-processing.  The following
+# should point to a file containing one line per tenant, with the
+# tenant name first, then whitespace, followed by the tenant ID.
+keystone_tenants_map: # keystoneTenantsMap="firstname,1234567890 secondname,0987654321"
+  - VAR_KEYSTONE_TENANTS_MAP
+
+# The reseller prefix is typically used by the Swift middleware to
+# keep accounts with different providers separate.  We would ideally
+# look this up dynamically from the Swift configuration.
+# The Keystone middlware defaults to the following value.
+reseller_prefix: VAR_RESELLER_PREFIX # resellerPrefix=AUTH_
+
+ring_path: VAR_RING_PATH # ringPath=/etc/swift
+
+# These will typically be read from /etc/swift/swift.conf.  If that
+# file cannot be opened, then the Swift library will log an error and
+# try to exit.  To run p-s-a-e as a user other than Swift, these
+# settings must be set to the same values as Swift itself, and the
+# above must point to an always-current readable copy of the rings.
+
+hash_path_prefix: VAR_HASH_PATH_PREFIX # hashPathPrefix=
+hash_path_suffix: VAR_HASH_PATH_SUFFIX # hashPathSuffix=

prometheus-openstack-exporter.yaml

 # Example configuration file for prometheus-openstack-exporter
-# Copyright (C) 2016 Canonical, Ltd.
+# Copyright (C) 2016-2019 Canonical, Ltd.
 #
 
 listen_port: 9183
@@ -9,6 +9,16 @@ cloud: mycloud
 openstack_allocation_ratio_vcpu: 2.5
 openstack_allocation_ratio_ram: 1.1
 openstack_allocation_ratio_disk: 1.0
+log_level: INFO
+
+# Configure the enabled collectors here.  Note that the Swift account
+# collector in particular has special requirements.
+enabled_collectors:
+  - cinder
+  - neutron
+  - nova
+  - swift
+#  - swift-account-usage
 
 # To export hypervisor_schedulable_instances metric set desired instance size
 schedulable_instance_size:
@@ -16,11 +26,38 @@ schedulable_instance_size:
     vcpu: 2
     disk_gbs: 20
 
+# Uncomment if the cloud doesn't provide cinder / nova volumes:
+#use_nova_volumes: False
+
+## Swift
+
 # There is no way to retrieve them using OpenStack APIs
 # For clouds deployed without swift, remove this part
 swift_hosts:
     - host1.example.com
     - host2.example.com
 
-# Uncomment if the cloud doesn't provide cinder / nova volumes:
-#use_nova_volumes: False
+# There is no API to ask Swift for a list of accounts it knows about.
+# Even if there were, Swift (in common case of Keystone auth, at
+# least) only knows them by the corresponding tenant ID, which would
+# be a less than useful label without post-processing.  The following
+# should point to a file containing one line per tenant, with the
+# tenant name first, then whitespace, followed by the tenant ID.
+keystone_tenants_map:
+
+# The reseller prefix is typically used by the Swift middleware to
+# keep accounts with different providers separate.  We would ideally
+# look this up dynamically from the Swift configuration.
+# The Keystone middlware defaults to the following value.
+reseller_prefix: AUTH_
+
+ring_path: /etc/swift
+
+# These will typically be read from /etc/swift/swift.conf.  If that
+# file cannot be opened, then the Swift library will log an error and
+# try to exit.  To run p-s-a-e as a user other than Swift, these
+# settings must be set to the same values as Swift itself, and the
+# above must point to an always-current readable copy of the rings.
+
+hash_path_prefix:
+hash_path_suffix:

prometheus_openstack_exporter.py

+prometheus-openstack-exporter
\ No newline at end of file

requirements.txt

+configparser==3.5
+flake8
+mock
+prometheus-client
+python-neutronclient
+python-novaclient
+swift

setup.py

@@ -5,6 +5,7 @@ from setuptools import setup
 def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
+
 setup(
     name="prometheus_openstack_exporter",
     version="0.0.4",
@@ -14,17 +15,20 @@ setup(
     keywords=["prometheus", "openstack", "exporter"],
     url="https://github.com/CanonicalLtd/prometheus-openstack-exporter",
     scripts=["prometheus-openstack-exporter"],
-    install_requires=["prometheus_client",
-                      "python-keystoneclient<=3.10.0",
-                      "python-novaclient==6.0.0",
-                      "python-neutronclient<=6.1.0",
-                      "python-cinderclient",
-                      "netaddr"],
+    install_requires=[
+        "prometheus_client",
+        "python-keystoneclient<=3.10.0",
+        "python-novaclient==6.0.0",
+        "python-neutronclient<=6.1.0",
+        "python-cinderclient",
+        "netaddr",
+        "swift",
+    ],
     long_description=read('README.md'),
     classifiers=[
         "Development Status :: 4 - Beta",
         "Topic :: System :: Networking :: Monitoring",
         "License :: OSI Approved :: "
-            "GNU General Public License v3 or later (GPLv3+)",
+        "GNU General Public License v3 or later (GPLv3+)",
     ],
 )

snap/.snapcraft/state

+!GlobalState
+assets:
+  build-packages: []
+  build-snaps: []

snapcraft.yaml → snap/snapcraft.yaml

 name: prometheus-openstack-exporter
-version: 0.0.8
+base: core
+version: git
 summary: Exposes high level OpenStack metrics to Prometheus.
 description: |
   Exposes high level OpenStack metrics to Prometheus
@@ -18,6 +19,7 @@ parts:
     source: .
     stage-packages:
       - libdb5.3
+      - liberasurecode-dev
   example-config:
     plugin: dump
     source: .

tests/test_SwiftAccountUsage.py

+import unittest
+
+from mock import Mock, call, patch
+
+from requests.structures import CaseInsensitiveDict
+
+import prometheus_openstack_exporter as poe
+
+
+class TestSwiftAccountUsage(unittest.TestCase):
+
+    @patch('prometheus_openstack_exporter.SwiftAccountUsage._get_account_ring')
+    @patch('prometheus_openstack_exporter.requests.head')
+    @patch('prometheus_openstack_exporter.config')
+    def test__get_account_usage(self, _config, _requests_head, _SwiftAccountUsage__get_account_ring):
+        s = poe.SwiftAccountUsage()
+        s.account_ring.get_nodes.return_value = (26701, [
+            {'device': 'sdb', 'id': 0, 'ip': '10.24.0.18', 'meta': u'', 'port': 6002,
+             'region': 1, 'replication_ip': '10.24.0.18', 'replication_port': 6002, 'weight': 100.0, 'zone': 1},
+            {'device': 'sdd', 'id': 50, 'ip': '10.24.0.71', 'meta': u'', 'port': 6002,
+             'region': 1, 'replication_ip': '10.24.0.71', 'replication_port': 6002, 'weight': 180.0, 'zone': 3},
+            {'device': 'sdi', 'id': 59, 'ip': '10.24.0.72', 'meta': u'', 'port': 6002,
+             'region': 1, 'replication_ip': '10.24.0.72', 'replication_port': 6002, 'weight': 360.0, 'zone': 2}
+            ])
+
+        response_mock = Mock()
+        response_mock.configure_mock(
+            status_code=204,
+            headers=CaseInsensitiveDict({'x-account-bytes-used': '368259416'}),
+            )
+        _requests_head.return_value = response_mock
+
+        # Assert that _get_account_ring does what we expect.
+        self.assertEqual(s._get_account_usage('AUTH_12bb569bf909441b90791482ae6f9ca9'), 368259416)
+
+        # Assert that _get_account_ring did it in the manner we expected.
+        s.account_ring.get_nodes.assert_called_once_with(account='AUTH_12bb569bf909441b90791482ae6f9ca9')
+        poe.requests.head.assert_called_once()
+        self.assertTrue(poe.requests.head.call_args in [
+            call('http://10.24.0.18:6002/sdb/26701/AUTH_12bb569bf909441b90791482ae6f9ca9'),
+            call('http://10.24.0.71:6002/sdd/26701/AUTH_12bb569bf909441b90791482ae6f9ca9'),
+            call('http://10.24.0.72:6002/sdi/26701/AUTH_12bb569bf909441b90791482ae6f9ca9'),
+            ])

tests/test_poe.py

+import unittest
+
+import mock
+
+import prometheus_openstack_exporter as poe
+
+
+class TestPrometheusOpenstackExporter(unittest.TestCase):
+    def test_data_gatherer_needed(self):
+        self.assertTrue(
+            poe.data_gatherer_needed(
+                {'enabled_collectors':
+                 ['cinder', 'neutron', 'nova', 'swift']}))
+        self.assertTrue(
+            poe.data_gatherer_needed(
+                {'enabled_collectors':
+                 ['cinder', 'neutron', 'nova', 'swift', 'swift-account-usage']}))
+        self.assertFalse(
+            poe.data_gatherer_needed(
+                {'enabled_collectors':
+                 ['swift-account-usage']}))
+        self.assertFalse(
+            poe.data_gatherer_needed(
+                {'enabled_collectors':
+                 ['swift', 'swift-account-usage']}))
+        self.assertEqual(
+            poe.data_gatherer_needed(
+                {'enabled_collectors':
+                 ['cinder', 'neutron', 'nova', 'swift']}),
+            set(['cinder', 'nova', 'neutron']),
+        )
+        self.assertEqual(
+            poe.data_gatherer_needed({}),
+            set(['cinder', 'nova', 'neutron']),
+        )
+
+    @mock.patch('prometheus_openstack_exporter.config')
+    def test_get_nova_info(self, config):
+        config.return_value = {}
+        prodstack = {'tenants': []}
+        nova = mock.Mock()
+        nova.aggregates = mock.MagicMock()
+        nova.flavors = mock.MagicMock()
+        nova.hypervisors = mock.MagicMock()
+        nova.servers = mock.MagicMock()
+        nova.servers.list = mock.MagicMock()
+        nova.services = mock.MagicMock()
+
+        data_gatherer = poe.DataGatherer()
+        data_gatherer._get_nova_info(nova, None, prodstack)
+
+        expected = [
+            mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'ACTIVE', 'all_tenants': '1'}),
+            mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'ERROR', 'all_tenants': '1'}),
+            mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'SHELVED_OFFLOADED', 'all_tenants': '1'}),
+            mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'SHUTOFF', 'all_tenants': '1'}),
+            mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'SUSPENDED', 'all_tenants': '1'}),
+            mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'VERIFY_RESIZE', 'all_tenants': '1'}),
+        ]
+        nova.servers.list.assert_has_calls(expected, any_order=True)
+
+        not_expected = mock.call(search_opts={'marker': '', 'limit': '100', 'status': 'BUILD', 'all_tenants': '1'})
+        self.assertTrue(not_expected not in nova.servers.list.call_args_list)

wrapper.sh

+#!/bin/sh
+
+prometheusDir='/etc/prometheus'
+configFile=${configFile:-"${prometheusDir}/prometheus-openstack-exporter.yaml"}
+listenPort=${listenPort:-9183}
+cacheRefreshInterval=${cacheRefreshInterval:-300}
+cacheFileName=${cacheFileName:-"$(mktemp -p /dev/shm/)"}
+cloud=${OS_REGION_NAME:-mycloud}
+vcpuRatio=${vcpuRatio:-1.0}
+ramRatio=${ramRatio:-1.0}
+diskRatio=${diskRatio:-1.0}
+enabledCollectors=${enabledCollectors:-cinder neutron nova}
+schedulableInstanceRam=${schedulableInstanceRam:-4096}
+schedulableInstanceVcpu=${schedulableInstanceVcpu:-2}
+schedulableInstanceDisk=${schedulableInstanceDisk:-20}
+useNovaVolumes=${useNovaVolumes:-True}
+swiftHosts=${swiftHosts:-host1.example.com host2.example.com}
+#keystoneTenantsMap="firstname,1234567890 secondname,0987654321"
+resellerPrefix=${resellerPrefix:-AUTH_}
+ringPath=${ringPath:-/etc/swift}
+#hashPathPrefix=
+#hashPathSuffix=
+
+if [ ! -e "${configFile}" ]; then
+    mkdir -p ${prometheusDir}
+    cp prometheus-openstack-exporter.sample.yaml ${configFile}
+    
+    sed -i "s|VAR_LISTEN_PORT|${listenPort}|g" 					${configFile}
+    sed -i "s|VAR_CACHE_REFRESH_INTERVAL|${cacheRefreshInterval}|g" 		${configFile}
+    sed -i "s|VAR_CACHE_FILE|${cacheFileName}|g" 				${configFile}
+    sed -i "s|VAR_CLOUD|${cloud}|g" 						${configFile}
+    sed -i "s|VAR_VCPU_RATIO|${vcpuRatio}|g" 					${configFile}
+    sed -i "s|VAR_RAM_RATIO|${ramRatio}|g" 					${configFile}
+    sed -i "s|VAR_DISK_RATIO|${diskRatio}|g"	 				${configFile}
+    sed -i "s|VAR_SCHEDULABLE_INSTANCE_RAM|${schedulableInstanceRam}|g" 	${configFile}
+    sed -i "s|VAR_SCHEDULABLE_INSTANCE_VCPU|${schedulableInstanceVcpu}|g" 	${configFile}
+    sed -i "s|VAR_SCHEDULABLE_INSTANCE_DISK|${schedulableInstanceDisk}|g" 	${configFile}
+    sed -i "s|VAR_USE_NOVA_VOLUMES|${useNovaVolumes}|g" 			${configFile}
+
+    for i in ${enabledCollectors}; do
+        sed -i "s/.*VAR_ENABLED_COLLECTORS/  - ${i}\n    - VAR_ENABLED_COLLECTORS/g" 	${configFile}
+    done
+    sed -i '/.*VAR_ENABLED_COLLECTORS.*/d'					${configFile} 
+
+    for i in ${swiftHosts}; do
+        sed -i "s/.*VAR_SWIFT_HOSTS/  - ${i}\n  - VAR_SWIFT_HOSTS/g" 		${configFile}
+    done
+    sed -i '/.*VAR_SWIFT_HOSTS.*/d'						${configFile} 
+
+    for i in ${keystoneTenantsMap}; do
+        tenantName=$(echo ${i} | cut -d',' -f1)
+        tenantId=$(  echo ${i} | cut -d',' -f2)
+        sed -i "s/.*VAR_KEYSTONE_TENANTS_MAP/  - ${tenantName} ${tenantId}\n  - VAR_KEYSTONE_TENANTS_MAP/g" ${configFile}
+    done
+    sed -i '/.*VAR_KEYSTONE_TENANTS_MAP.*/d' 					${configFile}
+
+    sed -i "s|VAR_RESELLER_PREFIX|${resellerPrefix}|g"	 			${configFile}
+    sed -i "s|VAR_RING_PATH|${ringPath}|g"	 				${configFile}
+    sed -i "s|VAR_HASH_PATH_PREFIX|${hashPathPrefix}|g"	 			${configFile}
+    sed -i "s|VAR_HASH_PATH_SUFFIX|${hashPathSuffix}|g"	 			${configFile}
+    
+    sed -i 's/VAR_.*//g'		 					${configFile}
+
+    touch ${cacheFileName}
+
+    cat ${configFile}
+fi
+
+/prometheus-openstack-exporter ${configFile}
+
+rm ${cacheFileName}
+
+exit 0