Some optimizations to improve throughput

opennebula-vm-etcd/put-vm-info-into-etcd.py

@@ -2,6 +2,7 @@ import pyone
 
 from enum import IntEnum
 from config import config, etcd_client
+from functools import reduce
 
 # How to get client secrets?
 # 1. Login to OpenNebula
@@ -14,24 +15,24 @@ one_client = pyone.OneServer(
     session=config['oca']['client_secrets']
 )
 
+host_pool = {
+    host.NAME: {
+        'name': host.NAME,
+        'id': host.ID,
+        'cluster': {
+            'name': host.CLUSTER,
+            'id': host.CLUSTER_ID
+        },
+        'vms': host.VMS.ID
+    }
+    for host in one_client.hostpool.info().HOST
+}
+
 
 def get_hostname_of_vm(vm_id):
-    host_pool = {
-        host.NAME: {
-            'name': host.NAME,
-            'id': host.ID,
-            'cluster': {
-                'name': host.CLUSTER,
-                'id': host.CLUSTER_ID
-            },
-            'vms': host.VMS.ID
-        }
-        for host in one_client.hostpool.info().HOST
-    }
     for hostname, host in host_pool.items():
         if vm_id in host['vms']:
             return host
-
     return None
 
 
@@ -115,11 +116,21 @@ def main():
     START_ID = -1  # First id whatever it is
     END_ID = -1  # Last id whatever it is
 
-    for VM_STATE in VM_STATES:
-        vm_pool = one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, VM_STATE)
-        for i, vm in enumerate(vm_pool.VM):
-            vm = VM(vm)
-            etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data())
+    # Get VMs in all kind of states
+
+    # vms is a list of lists
+    vms = [
+        one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, vm_state).VM
+        for vm_state in VM_STATES
+    ]
+    # Take out elements from nested lists and put them into the original list
+    # forming a nice flat list
+    vms = list(reduce(lambda n, n_1: n + n_1, vms))
+    print('Total VMs:', len(vms))
+    for i, _vm in enumerate(vms):
+        vm = VM(_vm)
+        etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data())
+        print(i, end=' ')
 
 
 if __name__ == "__main__":