Coverage for /home/agp/Documents/me/code/gutools/gutools/tools.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""This module contains basic methods and definitions that would be used
3by other `gutools` modules.
4"""
6import sys
7import os
8import asyncio
9import hashlib
10import random
11import inspect
12import types
13import fnmatch
14import re
15import yaml
16import time
17import json
18import stat
19import math
20import traceback
21import dateutil.parser as parser
22from urllib import parse
23from io import StringIO
24from functools import partial
25from collections.abc import Iterable
26from weakref import WeakValueDictionary
27from gutools.colors import *
29# -------------------------------------------------------
30# asyncio nested loops
31# -------------------------------------------------------
32# import nest_asyncio
33# nest_asyncio.apply()
36TYPES = []
37for k in types.__all__:
38 klass = getattr(types, k)
39 try:
40 if isinstance(TYPES, klass):
41 pass
42 TYPES.append(klass)
43 except Exception as why:
44 pass
45TYPES = tuple(TYPES)
47BASIC_TYPES = tuple([int, float, str, bytes, dict, list, tuple, ])
49# -----------------------------------------------------------
50# URI handling
51# -----------------------------------------------------------
52reg_uri = re.compile(
53 r"""
54 (
55 (?P<fscheme>
56 (?P<direction>[<|>])?(?P<scheme>[^:/]*))
57 ://
58 (?P<fservice>
59 (
60 (?P<auth>
61 (?P<user>[^:@/]*?)
62 (:(?P<password>[^@/]*?))?
63 )
64 @)?
65 (?P<host>[^@:/?]*)
66 )
67 (:(?P<port>\d+))?
68 )?
70 (?P<path>/[^?]*)?
71 (\?(?P<query>[^#]*))?
72 (\#(?P<fragment>.*))?
73 """,
74 re.VERBOSE | re.I | re.DOTALL)
76def parse_uri(uri, bind=None, **kw):
77 """Extended version for parsing uris:
79 Return includes:
81 - *query_*: dict with all query parameters splited
83 If `bind` is passed, *localhost* will be replace by argument.
85 """
86 m = reg_uri.match(uri)
87 if m:
88 for k, v in m.groupdict().items():
89 if k not in kw or v is not None:
90 kw[k] = v
91 if bind:
92 kw['host'] = kw['host'].replace('localhost', bind)
93 if kw['port']:
94 kw['port'] = int(kw['port'])
95 kw['address'] = tuple([kw['host'], kw['port']])
96 if kw['query']:
97 kw['query_'] = dict(parse.parse_qsl(kw['query']))
99 return kw
101def build_uri(fscheme='', direction='', scheme='',
102 host='', port='', path='',
103 query='', fragment=''):
104 """Generate a URI based on individual parameters"""
105 uri = ''
106 if fscheme:
107 uri += fscheme
108 else:
109 if not direction:
110 uri += scheme
111 else:
112 uri += f'{direction}{scheme}'
113 if uri:
114 uri += '://'
116 host = host or f'{uuid.getnode():x}'
117 uri += host
119 if port:
120 uri += f':{port}'
121 if path:
122 uri += f'{path}'
123 if query:
124 uri += f'?{query}'
125 if fragment:
126 uri += f'#{fragment}'
128 return uri
131# ---------------------------------------
133def snake_case(name):
134 s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
135 return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
137# containers
138def dset(d, key, value, *args, **kw):
139 if key not in d:
140 if args or kw:
141 value = value(*args, **kw)
142 d[key] = value
143 return d[key]
145# next_lid = random.randint(-10**5, 10**5)
146next_lid = random.randint(0, 10**5)
147def new_uid():
148 global next_lid
149 next_lid += 1
150 return next_lid # TODO: remove, just debuging
151 # return uidfrom(next_lid)
153def flatten(iterator, klass=None):
154 if not isinstance(iterator, Iterable):
155 yield iterator
156 return
157 # from https://stackoverflow.com/questions/2158395/flatten-an-irregular-list-of-lists
158 for item in iterator:
159 if isinstance(item, Iterable) and not isinstance(item, (str, bytes)):
160 yield from flatten(item)
161 elif klass is None or isinstance(item, klass):
162 yield item
164def deep_search(item, patterns, result=None):
165 """Explore deeply elements hierchachy searching items from
166 certain classes.
167 """
168 if result is None:
169 result = dict()
171 remain = [item]
173 # TODO: avoid circular references
174 used = set()
175 while remain:
176 item = remain.pop(0)
177 idx = id(item)
178 if idx in used:
179 # print('remain[{}] : {} : <{}> : {}'.format(len(remain), klass, holder, idx))
180 continue
181 used.add(idx)
183 klass = item.__class__.__name__
184 holder, idx = patterns.get(klass, (None, None))
186 # print('remain[{}] : {} : <{}> : {}'.format(len(remain), klass, holder, idx))
187 if idx:
188 # found an item of interest.
189 holder = holder or klass
190 key = getattr(item, idx)
191 if key is not None:
192 result.setdefault(holder, dict())[key] = item
193 # print('>> [{}][{}] = {}'.format(holder, key, klass))
194 foo = 1
196 if isinstance(item, dict):
197 # remain.extend(item.keys())
198 remain.extend(item.values())
199 elif isinstance(item, (list, tuple, set)):
200 remain.extend(item)
201 elif hasattr(item, '__dict__'):
202 remain.append(item.__dict__)
203 elif hasattr(item, '__slots__'):
204 remain.append(dict([(k, getattr(item, k, None)) for k in item.__slots__]))
205 else:
206 foo = 1 # discard item, we don't know how to go deeper
208 return result
210# --------------------------------------------------------------------
211# containers helpers
212# --------------------------------------------------------------------
213CONTAINERS = (dict, list, tuple)
214def walk(container, root=tuple(), includes={}, excludes={}):
215 """Walk recursive for an arbitrary container """
217 def buid_key(*keys):
218 "creare a key using the right factory"
220 # if factory in (tuple, ):
221 # keys = list(flatten(keys))
222 # return factory(keys)
223 # elif factory in (str, ):
224 # return '/'.join([factory(k) for k in keys])
225 # else:
226 # raise RuntimeError('Unknown factory type')
227 results = list(keys[0])
228 results.extend(keys[1:])
229 return tuple(results)
231 if isinstance(container, dict):
232 func = container.items
233 # yield '{}/'.format(root, ), '<{}>'.format(container.__class__.__name__)
234 yield buid_key(root, ), '<{}>'.format(container.__class__.__name__)
235 elif isinstance(container, (list, tuple)):
236 func = lambda : enumerate(container)
237 yield buid_key(root, ), '<{}>'.format(container.__class__.__name__)
238 else:
239 # TODO: apply includes/excludes
240 yield root, container # container is a single item
241 return
243 # EXCLUDES
244 recursive, match, same_klass = 0, 0, 0
245 to_exclude = set()
246 for key, item in func():
247 for klass, info in excludes.items():
248 if not isinstance(item, klass):
249 continue
250 same_klass += 1
251 for attr, filters in info.items():
252 value = getattr(item, attr)
253 if isinstance(value, CONTAINERS):
254 recursive += 1
255 continue
256 results = [f(value) for f in filters]
257 if any(match):
258 match += 1
259 to_exclude.add(key)
260 match += any(results)
261 if recursive == 0 and match == 1 and same_klass == 1:
262 # discard the whole container because there is 1 single element
263 # that match the excludes and the rest of element are related with
264 # different klasses (e.g. a tuple containing related information)
265 return
267 # TODO: coding 'includes' filters
269 # RECURSIVE exploring
270 for key, item in func():
271 if key in to_exclude:
272 continue
273 new_key = buid_key(root, key)
275 if isinstance(item, CONTAINERS):
276 yield from walk(item, new_key, includes, excludes)
277 else:
278 yield new_key, item
280def dive(container, key):
281 # if factory in (str, ):
282 # keys = [k for k in key.split('/') if k]
283 # else:
284 # keys = list(key)
285 keys = list(key)
286 # keys = [k if k != 'None' else None for k in keys]
287 while keys:
288 key = keys.pop(0)
289 if key in container:
290 container = container[key]
291 else:
292 break
293 assert len(keys) == 0
294 return container, key
296def divepush(container, key, item, default_container=dict):
297 parent = container
298 for key in key:
299 parent, container = container, container.setdefault(key, default_container())
300 parent[key] = item
302def rebuild(walk_iterator, default_converter=None, default_container=dict,
303 key_converters={}, container_converters={}, converters={}, ):
304 """rebuild a tree structure from walk result"""
305 result = None
306 # if factory in (str, ):
307 # key_converters = dict([re.compile(k, re.I | re.DOTALL), v] for k, v in key_converters.items())
309 def new_container(item):
310 """Try to get the container described in the item string:
312 <list> : return the list class
313 <dict> : return the dict class
315 """
317 if item and item[0] == '<' and item[-1] == '>':
318 klass = item[1:-1] # the klass name
319 try:
320 klass = eval(klass)
321 except Exception:
322 klass = default_container # as default building container for non-dict alike containers
323 return klass
325 convert = dict()
326 for key, item in walk_iterator:
327 klass = new_container(item) # try to guess if is a container
328 if klass is not None and not issubclass(klass, dict):
329 convert[key] = klass # for later converting the container
330 klass = default_container # as default building container for non-dict alike containers
332 if klass:
333 item = klass()
335 conv = converters.get(item.__class__, default_converter)
336 if conv:
337 item = conv(item)
339 if not key:
340 result = item
341 continue
343 assert result is not None, "root element must be found before any other"
344 divepush(result, key, item)
345 # we need to convert some nodes from botton-up to preserve
346 # indexing with keys as str, not integers in case of list/tuples
347 convert_keys = list(convert.keys())
348 # convert_keys.sort(key=lambda x: len(x.split('/')), reverse=True)
349 convert_keys.sort(key=len, reverse=True)
350 for key in convert_keys:
351 klass = convert[key]
352 # parent_key = key.split('/')
353 # parent_key.pop(-2)
354 # parent_key = '/'.join(parent_key)
355 parent_container, parent_key = dive(result, key[:-1])
356 container, child_key = dive(result, key)
357 keys = list(container.keys())
359 if issubclass(klass, (list, tuple)):
360 keys.sort(key=lambda x: int(x))
361 else:
362 keys.sort()
364 values = [container[k] for k in keys]
365 item = klass(values)
366 # chain converters
367 for conv in container_converters.get(klass, []):
368 item = conv(key, item)
370 # if parent_key:
371 # foo = 1
372 # else:
373 # # result = item
374 # foo = 1
376 parent_container[child_key] = item
378 return result
380def unflattern(iterator, keys, container=None):
381 """Build a structure info from a iterator, using some keys that
382 acts as indexes of the structure.
384 The value is taken from item itself.
385 """
386 container = container or dict()
387 for item in iterator:
388 parent, child = None, container
389 for key in keys:
390 if isinstance(item, (list, tuple)):
391 item = list(item)
392 key = item.pop(key)
393 elif isinstance(item, dict):
394 item = dict(item)
395 key = item.pop(key)
396 else:
397 key = getattr(item, key)
398 parent, child = child, child.setdefault(key, dict())
399 parent[key] = item
400 return container
402def serializable_container(container):
403 def _filter(container):
404 for path, v in walk(container):
405 if v in ('<tuple>', ):
406 yield path, '<list>'
407 elif v in (None, '<dict>', '<list>'):
408 yield path, v
409 elif v in ('<Config>', ):
410 yield path, '<dict>'
411 # must be at the end
412 elif isinstance(v, BASIC_TYPES):
413 yield path, v
414 else:
415 print(f"{RED}Don't know how to _filter{YELLOW} {path} = {v}{RESET}")
417 result = rebuild(_filter(container))
418 return result
420def update_container(container, value, *keys):
421 """Try to change a value in a container tree of nested list and dict
422 returning if tree has been changed or not.
423 """
424 def change(container, key, value):
425 modified = False
426 if isinstance(container, (list, )):
427 assert isinstance(key, int)
428 if value:
429 if key not in container:
430 container.append(key)
431 modified = True
432 else:
433 if key in container:
434 container.remove(key)
435 modified = True
436 else:
437 old = container.get(key)
438 if value != old:
439 container[key] = value
440 modified = True
441 return modified
444 for key in keys[:-1]:
445 if isinstance(container, (list, )):
446 key = int(key)
447 while len(container) < key:
448 container.append(None)
449 elif isinstance(container, (dict, )):
450 container = container.setdefault(key, dict())
451 else:
452 raise RuntimeError(f"Don't know how to handle {container.__class__} in container")
453 return change(container, keys[-1], value)
456def flatdict(container):
457 """Convert any structure in a flat dict that can be
458 rebuilt later.
460 flat = flatdict(container)
461 copy = rebuild(flat.items())
462 assert flat == copy
463 """
464 return dict([t for t in walk(container)])
466def diffdict(current, last):
467 """Compare to versions of the same flatdict container, giving:
469 - new items
470 - changed items
471 - deleted items
473 """
474 current_keys = set(current.keys())
475 last_keys = set(last.keys())
477 new_items = dict([(k, current[k]) for k in current_keys.difference(last_keys)])
478 deleted_items = dict([(k, last[k]) for k in last_keys.difference(current_keys)])
480 changed_items = dict()
481 for k in current_keys.intersection(last_keys):
482 c = current[k]
483 l = last[k]
484 if c != l:
485 changed_items[k] = (c, l)
487 return new_items, changed_items, deleted_items
490def sort_iterable(iterable, key=0):
491 "Specific function for sorting a iterable from a specific key position"
492 result = list()
493 iterable = list(iterable)
494 while iterable:
495 l = min([len(k[key]) if hasattr(k[key], '__len__') else 0 for k in iterable])
496 subset = list()
497 for i, k in reversed(list(enumerate(iterable))):
498 if not hasattr(k[key], '__len__') or len(k[key]) == l:
499 subset.append(k)
500 iterable.pop(i)
501 subset.sort(key=str, reverse=False)
502 result.extend(subset)
503 return result
505def merge(base, new, mode='add'):
507 def next_key_old(key):
508 key = key.split('/')
509 key[-1] = int(key[-1]) + 1
510 return '/'.join(key)
512 def next_key(key):
513 key = list(key)
514 key[-1] += 1 # must be an integer
515 return tuple(key)
517 def score(item):
518 "function for sorting tables"
519 return str(item)
521 def _merge(base, new):
522 base_ = list(walk(base))
523 new_ = list(walk(new))
525 base_.sort(key=score, reverse=False)
526 new_.sort(key=score, reverse=False)
528 last_container = []
529 last_parent = None
530 last_key = None
531 last_idx = 0
533 a = b = None
535 while base_ or new_:
536 if a is None and base_:
537 a = base_.pop(0)
538 if b is None and new_:
539 b = new_.pop(0)
541 if a and a[1] in ('<list>', ) or \
542 b and b[1] in ('<list>', ):
543 for b1 in last_container:
544 last_key = next_key(last_key)
545 yield (last_key, b1)
547 last_container = list()
548 last_parent = b and b[0]
550 if a is None and b is not None:
551 yield b
552 b = None
553 continue
554 if b is None and a is not None:
555 yield a
556 a = None
557 continue
559 a0 = str(a[0]) # to make comparissons possible
560 b0 = str(b[0])
561 if b0 < a0:
562 yield b
563 b = None
564 elif b == a:
565 yield b
566 a = b = None
567 elif b0 == a0:
568 # share the same key, but values differs
569 if mode in ('replace', ):
570 yield b
571 elif mode in ('add', ):
572 if isinstance(last_container, list):
573 last_key = a[0]
574 last_container.append(b[1])
575 yield a
576 else:
577 yield b
578 a = b = None
579 elif b0 > a0:
580 yield a
581 a = None
582 else:
583 raise RuntimeError('???')
585 foo = 1
587 # data = list()
588 from pprint import pprint
589 for pair in _merge(base, new):
590 print("-"*40)
591 print(f"{pair[0]}: {pair[1]}")
592 # data.append(pair)
593 # result = rebuild(data)
594 # pprint(result)
595 foo = 1
597 result = list(_merge(base, new))
599 # key_converters = {'None': None, }
601 result = sort_iterable(result)
602 result = rebuild(result)
604 return result
607def uidfrom(lid):
608 if not lid:
609 return lid
611 lid = str(lid)
612 lid = bytes(lid, encoding='UTF-8')
614 algo = 'md5'
615 assert algo in hashlib.algorithms_guaranteed
616 h = hashlib.new(algo)
617 h.update(lid)
618 uid = h.hexdigest()
620 return uid
622# https://stackoverflow.com/questions/6760685/creating-a-singleton-in-python
623class Singleton(type):
624 _instances = {}
625 _callargs = {}
626 def __call__(cls, *args, **kwargs):
627 if cls in cls._instances:
628 if args or kwargs and cls._callargs[cls] != (args, kwargs):
629 warn = """
630WARNING: Singleton {} called with different args\n
631 1st: {}
632 now: {}
633""".format(cls, cls._callargs[cls], (args, kwargs))
634 print(warn)
635 else:
636 cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
637 cls._callargs[cls] = (args, kwargs)
638 return cls._instances[cls]
642# #Python2
643# class MyClass(BaseClass):
644 # __metaclass__ = Singleton
646# #Python3
647# class MyClass(BaseClass, metaclass=Singleton):
648 # pass
650class Xingleton(type):
651 _instances = {}
652 def __call__(cls, *args, **kwargs):
653 __call__ = super(Xingleton, cls).__call__
654 __init__ = cls.__init__
655 args2 = prepare_call_args(__init__, *args, **kwargs)
656 # args2 = args
658 existing = cls._instances.get(cls) # don't use setdefault here
659 if existing is None:
660 existing = cls._instances[cls] = WeakValueDictionary()
662 instance = existing.get(args2)
663 if instance is None:
664 instance = existing[args2] = __call__(*args2)
665 return instance
667# ----------------------------------------------------------------------
668# instrospective and async calls
669# ----------------------------------------------------------------------
671class IntrospCaller(object):
672 """base class for instrospective calls and async calls.
674 Requires a context where locate:
675 - 'converters' dict for mapping arguments call with calleables or data
676 - 'call_keys' set to extract the key used to store deferred calls
678 Example:
680 # context for automatic IntrospCaller
681 converters = self.context.setdefault('converters', {})
682 converters['reqId'] = self._next_rid
683 converters['contract'] = self._get_contract
685 self.context.setdefault('call_keys', set()).update(['reqId', ])
687 for symbol in ('NQ', 'ES', 'DAX'):
688 self.make_call(self.protocol.client.reqContractDetails)
690 It will make 3 calls that will use 'reqId' parameter as key
692 In order to drop the cache when request is done user must explicit invoke
694 self._drop_call(kw)
696 where kw contains the same parameter that has been used to make the call
698 """
700 def __init__(self, context=None):
701 self.context = context if context is not None else dict()
702 self._make_request_cache = dict()
703 self._calls = dict()
705 def make_call(self, func, **ctx):
706 frame = inspect.currentframe().f_back
707 env = dict(frame.f_locals)
708 context = {}
709 while frame and not context:
710 # converters = frame.f_locals.get('kw', {}).get('__context__', {}).get('converters', {})
711 for name in ('context', 'ctx', ):
712 context = frame.f_locals.get(name, {})
713 if context:
714 break
715 frame = frame.f_back
716 converters = context.get('converters', {})
718 # execute cached strategy
719 strategy = self._get_strategy(func, env, converters)
720 kw = execute_strategy(strategy, env, **converters)
721 keys = self._set_call(func, kw)
722 return func(**kw), keys
724 def _get_strategy(self, func, env, converters):
725 strategy = self._make_request_cache.get(func)
726 if strategy is not None:
727 return strategy
729 strategy = strategy_call(func, env, **converters)
730 self._make_request_cache[func] = strategy
731 return strategy
733 def _set_call(self, func, kw):
734 # store call parameters for later use
735 for key in self.context['call_keys'].intersection(kw):
736 value = kw[key]
737 self._calls[value] = func, value, kw
739 # just 1 iteration, maybe more than one for advanced uses
740 return func, value, kw
742 def _drop_call(self, kw):
743 result = []
744 for key in self.context['call_keys'].intersection(kw):
745 result.append(self._calls.pop(kw[key], None))
746 return result
750def strategy_call(func, env, **converters):
751 """Try to find matching calling arguments making a
752 deep search in `**kw` arguments."""
753 func_info = inspect.getfullargspec(func)
754 callargs = list(func_info.args)
755 defaults = list(func_info.defaults or [])
756 annotations = func_info.annotations or {}
757 strategy = dict()
758 while callargs:
759 attr = callargs.pop(0)
760 klass = annotations.get(attr)
761 # find a object that match attr
762 best_name, best_score = None, 0
763 for name, value in env.items():
764 if name in strategy:
765 continue
766 # d1 = likelyhood(attr, name)
767 # d2 = likelyhood2(attr, name)
768 # print("<{}, {}> : {} {}".format(attr, name, d1, d2))
769 score = (likelyhood4(attr, name) + \
770 (klass in (None, value.__class__))) / 2
771 if 0.50 <= score > best_score:
772 best_name, best_score = name, score
773 if best_name:
774 strategy[attr] = ('env', best_name)
775 continue
776 # otherwise we need to use converters
777 # try to find the best converter possible
778 # same or similar names, same return values is provided by signature
779 best_name, best_score = None, 0
780 for name, conv in converters.items():
781 conv_info = inspect.getfullargspec(conv)
782 ret = conv_info.annotations.get('return')
783 if ret in (None, klass):
784 score = likelyhood(attr, name)
785 if score > best_score:
786 best_name, best_score = name, score
787 if best_name:
788 strategy[attr] = ('conv', best_name)
790 if isinstance(func, types.MethodType):
791 bound = func_info.args[0]
792 assert bound == 'self' # is a convenience criteria
793 strategy.pop(bound, None)
795 # self._make_request_cache[request] = strategy
796 return strategy
798def execute_strategy(strategy, env, **converters):
799 kw2 = dict()
800 for attr, (where, best) in strategy.items():
801 if where in ('env'):
802 value = env.get(best)
803 elif where in ('conv'):
804 conv = converters[best]
805 args, kw = prepare_call(conv, **env)
806 value = conv(*args, **kw)
807 kw2[attr] = value
808 return kw2
810def update_context(context, *args, **kw):
811 __avoid_nulls__ = kw.pop('__avoid_nulls__', True)
813 if __avoid_nulls__:
814 for k, v in kw.items():
815 if v is not None:
816 context[k] = v
817 else:
818 context.update(kw)
820 for item in args:
821 if isinstance(item, dict):
822 d = item
823 elif hasattr(item, 'as_dict'):
824 d = item.as_dict()
825 elif hasattr(item, '__dict__'):
826 d = item.__dict__
827 elif hasattr(item, '__getstate__'):
828 d = item.__getstate__()
829 else:
830 d = dict()
831 for k in dir(item):
832 if k.startswith('_'):
833 continue
834 v = getattr(item, k)
835 if v.__class__.__name__ == 'type' or isinstance(v, TYPES):# (types.FunctionType, types.MethodType, types.MethodWrapperType, types.BuiltinFunctionType)):
836 continue
837 d[k] = v
839 if __avoid_nulls__:
840 for k, v in d.items():
841 if v is not None:
842 context[k] = v
843 else:
844 context.update(d)
846def prepare_call(func, *args, **kw):
847 # collect available variables from stack
848 __max_frames_back__ = kw.pop('__max_frames_back__', 1)
849 frame = sys._getframe(1)
850 frameN = sys._getframe(__max_frames_back__)
851 _locals = list()
852 while __max_frames_back__ > 0 and frame:
853 _locals.append(frame.f_locals)
854 __max_frames_back__ -= 1
855 frame = frame.f_back
856 _locals.reverse()
857 kw0 = dict()
858 for st in _locals:
859 for item in st.values():
860 update_context(kw0, item)
861 update_context(kw0, st)
862 kw0.update(kw)
864 # try to match function calling args
865 info = inspect.getfullargspec(func)
866 kw2 = dict()
867 args = list(args)
868 callargs = list(info.args)
869 defaults = list(info.defaults or [])
870 # remove self for MethodType, and __init__
871 if isinstance(func, types.MethodType) or \
872 func.__name__ in ('__init__', ) or \
873 func.__class__.__name__ in ('type', ): # klass.__call__
874 if callargs[0] == 'self': # is a convenience criteria
875 callargs.pop(0)
876 kw0.pop('self', None)
878 while len(defaults) < len(callargs):
879 defaults.insert(0, None)
881 while callargs:
882 attr = callargs.pop(0)
883 value = defaults.pop(0)
884 if attr in kw0:
885 value = kw0.pop(attr)
886 if args:
887 if id(value) == id(args[0]):
888 args.pop(0)
889 elif args:
890 value = args.pop(0)
891 kw2[attr] = value
893 if not info.varargs:
894 if len(args) > 0:
895 raise RuntimeError('too many positional args ({}) for calling {}(...)'.format(args, func.__name__))
896 if info.varkw:
897 kw2.update(kw0)
898 # if isinstance(func, types.MethodType):
899 # bound = info.args[0]
900 # assert bound == 'self' # is a convenience criteria
901 # kw2.pop(bound)
902 return args, kw2
904def prepare_call_args(func, *args, **kw):
905 args2, kw2 = prepare_call(func, *args, **kw)
906 kw2.pop('self', None)
907 info = inspect.getfullargspec(func)
908 args2.extend([k for k in info.args if k in kw2])
909 return tuple([kw2[k] for k in args2])
912def _call(func, *args, **kw):
913 # 1. try to execute directly
914 if len(args) == 1:
915 if isinstance(args[0], dict):
916 args[0].update(kw)
917 args, kw = [], args[0]
918 elif isinstance(args[0], (list, tuple)):
919 args = args[0]
921 try:
922 args2, kw2 = prepare_call(func, *args, **kw)
923 return func(*args2, **kw2)
924 except Exception as why: # TODO: which is the right exception here? (missing args)
925 # TypeError
926 traceback.print_exc()
928 print(f"{YELLOW}ERROR: _call({func}) -> {BLUE}{why}{RED}")
929 traceback.print_exc(file=sys.stdout)
930 # exc_info = sys.exc_info()
931 # tb = exc_info[-1]
932 # tb.tb_next
933 # frame = tb.tb_next.tb_frame
935 # print(traceback.print_exception(*exc_info))
936 print(f"{RESET}")
937 # del exc_info
938 foo = 1
940 # 2. try to find calling arguments from passed dict as env
941 # TODO: include default **converters as well?
943 strategy = strategy_call(func, kw)
944 kw2 = execute_strategy(strategy, kw)
945 try:
946 return func(**kw2)
947 except Exception as why: # TODO: which is the right exception here? (missing args)
948 foo = 1
950def async_call(func, *args, **kw):
951 main = _call(func, *args, **kw)
952 assert asyncio.iscoroutine(main)
953 asyncio.run(main)
955# ---------------------------------------------------------------------
956# A Dynamic Programming based Python program for edit distance problem
957# ---------------------------------------------------------------------
958def editDistDP(name1, name2):
959 m, n = len(name1), len(name2)
960 # Create a table to store results of subproblems
961 dp = [[0 for x in range(n+1)] for x in range(m+1)]
963 # Fill d[][] in bottom up manner
964 for i in range(m+1):
965 for j in range(n+1):
967 # If first string is empty, only option is to
968 # insert all characters of second string
969 if i == 0:
970 dp[i][j] = j # Min. operations = j
972 # If second string is empty, only option is to
973 # remove all characters of second string
974 elif j == 0:
975 dp[i][j] = i # Min. operations = i
977 # If last characters are same, ignore last char
978 # and recur for remaining string
979 elif name1[i-1] == name2[j-1]:
980 dp[i][j] = dp[i-1][j-1]
982 # If last character are different, consider all
983 # possibilities and find minimum
984 else:
985 dp[i][j] = 1 + min(dp[i][j-1], # Insert
986 dp[i-1][j], # Remove
987 dp[i-1][j-1]) # Replace
989 return dp[m][n]
991def wlikelyhood(set1, set2, **weights):
992 W = 0
993 S = 0
994 for k, w in weights.items():
995 s1, s2 = set1.get(k, []), set2.get(k, [])
996 if s1 or s2:
997 # only weight average when any set is not empty
998 W += w
999 s = likelyhood4(s1, s2)
1000 S += w * s
1002 return S / (W or 1)
1005def likelyhood4(set1, set2):
1006 set1, set2 = list(set1), list(set2)
1007 N = (len(set1) + len(set2)) or 1
1008 S = 0
1009 def check(s1, s2):
1010 s = 0
1011 while s1:
1012 item = s1.pop(0)
1013 if item in s2:
1014 s2.remove(item)
1015 s += 2
1016 else:
1017 s -= 1
1018 return s
1019 S = check(set1, set2) + check(set2, set1)
1020 return S / N
1022def likelyhood3(set1, set2):
1023 n = min(len(set1), len(set2))
1024 if n:
1025 return 1 - editDistDP(set1, set2) / n
1026 return 0
1028def likelyhood2(name1, name2):
1029 n = min(len(name1), len(name2))
1030 return 1 - editDistDP(name1.lower(), name2.lower()) / n
1032def likelyhood(name1, name2):
1033 if len(name1) > len(name2):
1034 name1, name2 = list(name2.lower()), list(name1.lower())
1035 else:
1036 name1, name2 = list(name1.lower()), list(name2.lower())
1038 n = len(name1)
1039 score = 0
1040 while name1:
1041 a = name1.pop(0)
1042 if a in name2:
1043 idx = name2.index(a)
1044 name2.pop(idx)
1045 else:
1046 idx = n
1047 score += 1 / (1 + idx) ** 2
1049 return score / n
1053def get_calling_function(level=1):
1054 """finds the calling function in many decent cases."""
1055 # stack = inspect.stack(context)
1056 # fr = sys._getframe(level) # inspect.stack()[1][0]
1057 stack = inspect.stack()
1058 while level < len(stack):
1059 fr = stack[level][0]
1060 co = fr.f_code
1061 for get in (
1062 lambda: fr.f_globals[co.co_name],
1063 lambda: getattr(fr.f_locals['self'], co.co_name),
1064 lambda: getattr(fr.f_locals['cls'], co.co_name),
1065 lambda: fr.f_back.f_locals[co.co_name], # nested
1066 lambda: fr.f_back.f_locals['func'], # decorators
1067 lambda: fr.f_back.f_locals['meth'],
1068 lambda: fr.f_back.f_locals['f'],
1069 ):
1070 try:
1071 func = get()
1072 except (KeyError, AttributeError):
1073 pass
1074 else:
1075 if func.__code__ == co:
1076 return func
1077 level += 1
1078 raise AttributeError("func not found")
1080def retry(delay=0.1):
1081 """Retray the same call where the funcion is invokerd but
1082 a few instant later"""
1083 frame = inspect.currentframe().f_back
1084 calling_args = frame.f_code.co_varnames[:frame.f_code.co_argcount]
1085 calling_args = [frame.f_locals[k] for k in calling_args]
1086 func = get_calling_function(2)
1087 if isinstance(func, types.MethodType):
1088 calling_args.pop(0)
1090 loop = asyncio.get_event_loop()
1091 loop.call_later(0.2, func, *calling_args)
1093 foo = 1
1095async def add_to_loop(*aws, delay=0.1, loop=None):
1096 for coro in aws:
1097 asyncio.ensure_future(coro, loop=loop)
1098 await asyncio.sleep(delay, loop=loop)
1100# --------------------------------------------------
1101# RegExp operations
1102# --------------------------------------------------
1103def _prepare_test_regext(regexp=None, wildcard=None, test=None):
1104 test = test or list()
1106 if isinstance(regexp, (list, tuple)):
1107 test.extend(regexp)
1108 else:
1109 test.append(regexp)
1111 if not isinstance(wildcard, (list, tuple)):
1112 wildcard = [wildcard]
1113 for wc in wildcard:
1114 if wc and isinstance(wc, str):
1115 wc = fnmatch.translate(wc)
1116 test.append(wc)
1118 test = [re.compile(m).search for m in test if m]
1120 return test # you can compile a compiled expression
1122def _find_match(string, test):
1123 for match in test:
1124 m = match(string)
1125 if m:
1126 return m
1128def _return_matched_info(m, info):
1129 if info == 'd':
1130 return m.groupdict()
1131 elif info == 'g':
1132 return m.groups()
1134def _return_unmatched(info):
1135 if info == 'd':
1136 return dict()
1137 elif info == 'g':
1138 return list()
1140def parse_string(string, regexp=None, wildcard=None, info=None):
1141 test = _prepare_test_regext(regexp, wildcard)
1142 m = _find_match(string, test)
1143 if m:
1144 return _return_matched_info(m, info)
1145 return _return_unmatched(info) # to return cpmpatible values in upper stack
1147def parse_date(date):
1148 if isinstance(date, str):
1149 return parser.parse(date)
1150 return date
1152# --------------------------------------------------
1153# Speed control
1154# --------------------------------------------------
1155class SpeedControl(list):
1156 """Calculate the next pause needed to keep an average
1157 speed limit between requests to some service.
1158 """
1159 def __init__(self, lapse=0, N=1, *args, **kwargs):
1160 self.lapse = lapse
1161 if lapse:
1162 N = N or int(100/(lapse or 1)) + 1
1163 N = min(max(N, 10), 100)
1164 else:
1165 N = 1
1166 self.N = N
1168 @property
1169 def pause(self):
1170 "The next pause before make the next request"
1171 N = len(self)
1172 t1 = time.time()
1173 self.append(t1)
1174 t0 = self.pop(0) if len(self) > self.N else self[0]
1175 t1p = self.lapse * N + t0
1176 return t1p - t1
1178 @property
1179 def speed(self):
1180 "Return the current average speed"
1181 elapsed = self[-1] - self[0]
1182 if elapsed > 0:
1183 return len(self) / elapsed
1184 return 0
1185 # return float('inf')
1187 @property
1188 def round_speed(self):
1189 # return math.ceil(self.speed * 100) / 100
1190 return int(self.speed * 100) / 100
1195# --------------------------------------------------
1196# Progress
1197# --------------------------------------------------
1198class ProgressControl(list):
1199 def __init__(self, d0, d1):
1200 self.d0 = d0
1201 self.range_ = d1 - d0
1202 # assert self.range_ > 0
1204 def progress(self, d):
1205 return int(10000*(d - self.d0)/self.range_) / 100
1208# --------------------------------------------------
1209# File perations
1210# --------------------------------------------------
1211def fileiter(top, regexp=None, wildcard=None, info=None, relative=False):
1212 """Iterate over files
1213 info == 'd' : returns filename, regexp.groupdict()
1214 info == 'g' : returns filename, regexp.groups()
1215 info == None: : returns filename
1217 Allow single expressions or iterator as regexp or wildcard params
1218 """
1219 test = _prepare_test_regext(regexp, wildcard)
1220 for root, _, files in os.walk(top):
1221 for name in files:
1222 filename = os.path.join(root, name)
1223 m = _find_match(filename, test)
1224 if m:
1225 if relative:
1226 filename = filename.split(top)[-1][1:]
1227 m = _return_matched_info(m, info)
1228 if m:
1229 yield filename, m
1230 else:
1231 yield filename
1232 foo = 1
1234def copyfile(src, dest, override=False):
1235 if os.path.exists(dest) and not override:
1236 return
1238 folder = os.path.dirname(dest)
1239 if not os.path.exists(folder):
1240 os.makedirs(folder)
1241 with open(src, 'rb') as s:
1242 with open(dest, 'wb') as d:
1243 d.write(s.read())
1245 st = os.stat(src)
1246 os.chmod(dest, stat.S_IMODE(st.st_mode))
1248def expandpath(path):
1249 if path:
1250 path = os.path.expanduser(path)
1251 path = os.path.expandvars(path)
1252 path = os.path.abspath(path)
1253 while path[-1] == '/':
1254 path = path[:-1]
1255 return path
1257def load_config(path):
1258 raise DeprecationWarning()
1259 loader = {
1260 'yaml': yaml.load,
1261 'json': json.load,
1262 }
1263 config = dict()
1264 if os.path.exists(path):
1265 ext = os.path.splitext(path)[-1][1:]
1266 loader = loader.get(ext, loader['yaml'])
1267 with open(path, 'r') as f:
1268 config = loader(f) or config
1269 return config
1271def save_config(config, path):
1272 raise DeprecationWarning()
1274 saver = {
1275 'yaml': partial(yaml.dump, default_flow_style=False),
1276 'json': json.dump,
1277 }
1278 ext = os.path.splitext(path)[-1][1:]
1279 saver = saver.get(ext, saver['yaml'])
1280 with open(path, 'w') as f:
1281 config = saver(config, f)
1283def yaml_decode(raw: str):
1284 fd = StringIO(raw)
1285 return yaml.load(fd)
1287def yaml_encode(item: str):
1288 fd = StringIO()
1289 yaml.dump(item, fd, default_flow_style=False)
1290 return fd.getvalue()
1292def identity(item):
1293 "convenience function"
1294 return item
1296def test_pid(pid):
1297 try:
1298 pid = int(pid)
1299 os.kill(pid, 0)
1300 except OSError:
1301 return False
1302 else:
1303 return True
1305# - End -