experimental watch_prefix with new timeout mechanism added

etcd_wrapper.py

@@ -3,6 +3,7 @@ import json
 import signal
 import threading
 import time
+import queue
 
 from collections import namedtuple
 from dataclasses import dataclass
@@ -52,41 +53,49 @@ class Etcd3Wrapper(object):
             yield EtcdEntry(*entry[::-1], value_in_json=value_in_json)
 
     # def watch_prefix(self, *args, timeout=0, value_in_json=False, **kwargs):
+    #     def raise_timeout(signum, frame):
+    #         raise TimeoutError("timeout")
+        
+    #     r, _ = self.client.watch_prefix(*args, **kwargs)
+
     #     if timeout > 0:
     #         signal.signal(signal.SIGALRM, raise_timeout)
+
     #     while True:
     #         try:
-    #             if timeout > 0:
+    #             signal.alarm(timeout)
-    #                 signal.alarm(timeout)
-    #             r, _ = self.client.watch_prefix(*args, **kwargs)
 
     #             for event in r:
     #                 signal.alarm(0)
-    #                 # if e.value is None don't propagate its value
+    #                 if event.value:
-    #                 if event.value is None:
+    #                     event = EtcdEntry(event, event.value, value_in_json=value_in_json)
-    #                     continue
+    #                     signal.alarm(timeout)
-    #                 event = EtcdEntry(event, event.value, value_in_json=value_in_json)
+    #                     yield event
-    #                 signal.alarm(timeout)
+    #                 else:
-    #                 yield event
+    #                     signal.alarm(timeout)
     #         except TimeoutError:
     #             _value = {"status": "TIMEOUT"}
     #             yield EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"), value=str.encode(json.dumps(_value)))
 
-    def watch_prefix(self, key_prefix, timeout=0, value_in_json=False, **kwargs):
+    def watch_prefix(self, key_prefix, timeout,
+                     value_in_json=False, **kwargs):
+        q = queue.Queue()
         r, _ = self.client.watch_prefix(key_prefix, **kwargs)
-        if timeout > 0:
+        t = threading.Thread(target=watch_prefix_generator,
-            t = threading.Thread(target=create_timeout_event,
+                             args=(value_in_json, r, q))
-                                 args=(timeout, key_prefix, self.client))
+        t.start()
-            t.start()
 
-        for event in r:
+        while True:
-            if event.value:
+            try:
-                event = EtcdEntry(event, event.value, value_in_json=value_in_json)
+                v = q.get(timeout=timeout)
-                yield event        
+                yield v
+            except queue.Empty:
+                _value = {"status": "TIMEOUT"}
+                e = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"), value=str.encode(json.dumps(_value)))
+                yield e
 
-def create_timeout_event(timeout, key_prefix, etcd_client):
+def watch_prefix_generator(value_in_json, generator, q):
-    while True:
+    for event in generator:
-        time.sleep(timeout)
+        if event.value:
-        _value = {"status": "TIMEOUT", "last_timeout": datetime.utcnow().isoformat()}
+            event = EtcdEntry(event, event.value)
-        _key = path.join(f"{key_prefix}", "TIMEOUT")
+            q.put(event)        
-        etcd_client.put(_key.encode("utf-8"), json.dumps(_value))