Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2import asyncio 

3import random 

4import re 

5import os 

6import threading 

7import traceback 

8from itertools import chain 

9from collections import OrderedDict 

10from time import time, sleep 

11from select import select 

12import socket 

13import operator 

14from weakref import WeakKeyDictionary, WeakValueDictionary 

15 

16# renders 

17import graphviz 

18from jinja2 import Environment, PackageLoader, select_autoescape 

19 

20# mine 

21from gutools.tools import _call, walk, rebuild, merge, parse_uri, flatten, \ 

22 IntrospCaller 

23from gutools.utests import speed_meter 

24from gutools.recording import Recorder 

25 

26CO_COROUTINE = 128 

27 

28STATE_INIT = 'INIT' 

29STATE_READY = 'READY' 

30STATE_END = 'END' 

31EVENT_TERM = '<term>' # term alike 

32EVENT_QUIT = '<quit>' # kill alike, but a chance to do something before die 

33 

34MERGE_ADD = 'add' 

35MERGE_REPLACE_EXISTING = 'replace_existing' 

36MERGE_REPLACE_ALL = 'replace_all' 

37 

38NO_STATE = [] 

39QUITE_STATE = [[], [], []] 

40 

41GROUP_ENTRY = 0 

42GROUP_DO = 1 

43GROUP_EXIT = 2 

44 

45def bind(func_name, context): 

46 func = func_name and context[func_name] 

47 # TODO: assert is calleable 

48 # func.__binded__ = True # just a tag for now 

49 return func 

50 

51def precompile(exp): 

52 code = compile(exp, '<input>', mode='eval') 

53 return code 

54 

55def _merge(states, transitions, mode=MERGE_ADD): 

56 """Merge states and transitions to create hierarchies of layers""" 

57 _states = states.pop(0) if states else dict() 

58 _transitions = transitions.pop(0) if transitions else dict() 

59 

60 if mode in (MERGE_ADD, ): 

61 pass 

62 elif mode in (MERGE_REPLACE_EXISTING, ): 

63 for st in states: 

64 for state, new_functions in st.items(): 

65 _states.pop(state, None) 

66 

67 for tr in transitions: 

68 for source, new_info in tr.items(): 

69 info = _transitions.setdefault(source, dict()) 

70 for event, new_trx in new_info.items(): 

71 trx = info.setdefault(event, list()) 

72 for new_t in new_trx: 

73 for i, t in reversed(list(enumerate(trx))): 

74 if t[:1] == new_t[:1]: 

75 trx.pop(i) 

76 

77 elif mode in (MERGE_REPLACE_ALL, ): 

78 states.clear() 

79 for tr in transitions: 

80 for source, new_info in tr.items(): 

81 _transitions.pop(source, None) 

82 else: 

83 raise RuntimeError(f"Unknown '{mode}' MERGE MODE") 

84 

85 # merge states 

86 for st in states: 

87 for state, new_functions in st.items(): 

88 functions = _states.setdefault(state, list([[], [], []])) 

89 for i, func in enumerate(new_functions): 

90 for f in func: 

91 if f not in functions[i]: 

92 functions[i].append(f) 

93 

94 # merge transitions 

95 for tr in transitions: 

96 for source, new_info in tr.items(): 

97 info = _transitions.setdefault(source, dict()) 

98 for event, new_trx in new_info.items(): 

99 trx = info.setdefault(event, list()) 

100 for new_t in new_trx: 

101 for t in trx: 

102 if t[:2] == new_t[:2]: 

103 t[-1].extend(new_t[-1]) 

104 break 

105 else: 

106 trx.append(new_t) 

107 

108 

109 

110 

111 return _states, _transitions 

112 

113DEFAULT_STYLES = { 

114 'invisible': {'color': 'grey', 'shape': 'point'}, 

115 'timer': {'color': 'darkgreen', 'fontcolor': 'darkgreen', 'style': 'dashed'}, 

116 'rotation': [{'color': 'red', 'style': 'solid', }, {'color': 'orange', 'style': 'solid', }], 

117 'no_precond': {'color': 'blue', 'fontcolor': 'blue', 'style': 'solid'}, 

118 'no_label': {'color': 'gray', 'style': 'dashed'}, 

119} 

120 

121class Layer(IntrospCaller): 

122 """ 

123 States: 

124 

125 [entry, do, each, exit ] 

126 

127 Trasition: 

128 [ state, new_state, preconditions, functions ] 

129  

130 Layer definition is defined using `_setup_xxx` member functions 

131 that provide states, transitions sets. 

132  

133 boths sets may be combined using: 

134  

135 - MERGE_REPLACE_ALL: replace current state with provided one. 

136 - MERGE_REPLACE_EXISTING: only replace conflicting states/transitions. 

137 - MERGE_ADD: extend any current states/transitions sets. 

138  

139 `_setup_xxx` methods are called sorted by nane within Layer. 

140  

141 """ 

142 

143 def __init__(self, states=None, transitions=None, context=None): 

144 super().__init__(context) 

145 

146 # self._state = None 

147 self.states = states if states is not None else dict() 

148 self.transitions = transitions if transitions is not None else dict() 

149 

150 self.reactor = None 

151 self.state = None 

152 

153 # Setup layer logic 

154 for name, states, transitions, mode, _ in self._get_layer_setups(): 

155 self._merge(states, transitions, mode) 

156 

157 def start(self, **kw): 

158 """Call when layer starts""" 

159 

160 def bye(self, **kw): 

161 """Request the layer to term sending a EVENT_TERM event""" 

162 self.reactor.publish(EVENT_TERM, None) 

163 

164 def term(self, key, **kw): 

165 """Request the layer to quit sending a EVENT_QUIT event""" 

166 self.reactor.publish(EVENT_QUIT, None) 

167 

168 def quit(self, key, **kw): 

169 print(f" > Forcing {self.__class__.__name__} fo QUIT. Detaching from Reactor") 

170 self.reactor.detach(self) 

171 

172 def _compile(self, states=None, transitions=None): 

173 """Precompile preconditions expressions and function calls as possible. 

174 """ 

175 states = self.states if states is None else states 

176 transitions = self.transitions if transitions is None else transitions 

177 

178 # prepare context with function as well 

179 context = dict([(k, getattr(self, k)) for k in [k for k in dir(self) if k[0] != '_']]) 

180 

181 # bind transitions functions 

182 for source, info in transitions.items(): 

183 for event, trans in info.items(): 

184 if len(trans) > 3: 

185 continue # it's already compiled 

186 for trx in trans: 

187 trx.append(list()) 

188 target, precond, functions, comp_precond = trx 

189 for i, func in enumerate(precond): 

190 comp_precond.append(precompile(func)) 

191 for i, func in enumerate(functions): 

192 functions[i] = bind(func, context) 

193 

194 # bind state functions 

195 for grp_functions in states.values(): 

196 for group, functions in enumerate(grp_functions): 

197 for i, func in enumerate(functions): 

198 functions[i] = bind(func, context) 

199 foo = 1 

200 

201 def _merge(self, states, transitions, mode=MERGE_ADD): 

202 """Merge states and transitions to create hierarchies of layers""" 

203 _merge([self.states, states], [self.transitions, transitions], mode=mode) 

204 

205 def _trx_iterator(self): 

206 """evaluate all(event, state) pairs that the layer can process""" 

207 for source, info in self.transitions.items(): 

208 for event, transitions in info.items(): 

209 yield source, event, transitions 

210 

211 def graph(self, graph_cfg=None, styles=None, isolated=True, view=False, include=None, skip=None, name=None, format='svg', path=None): 

212 """Create a graph of the layer definition""" 

213 

214 nodes = set() 

215 include = include or [] 

216 skip = skip or [] 

217 path = path or os.path.join(os.path.abspath(os.curdir), 'stm') 

218 os.makedirs(path, exist_ok=True) 

219 name = name or f"{self.__class__.__name__}" 

220 

221 def new_aux_node(style): 

222 aux_node = random.randint(0, 10**8) 

223 name = f"aux_{aux_node}" 

224 node = dg.node(name, **style) 

225 return name 

226 

227 def next_style(stls): 

228 st = stls.pop(0) 

229 stls.append(st) 

230 return st 

231 

232 def new_node(key, functions=None): 

233 key_ = f"{prefix}_{key}" 

234 if key_ not in nodes: 

235 nodes.add(key_) 

236 return dg.node(name=key_, label=key) 

237 

238 def edge_attrs(source, event, target, precond, functions): 

239 

240 if event is None: 

241 label = [] 

242 else: 

243 label = [f"{event} ->"] 

244 

245 for i, exp in enumerate(precond): 

246 label.append(f"[{exp}]") 

247 for i, func in enumerate(functions): 

248 if not isinstance(func, str): 

249 func = func.__name__ 

250 label.append(f"{i}: {func}()") 

251 label = '\n'.join(label) 

252 

253 if label.startswith('each'): 

254 style = styles['timer'] 

255 elif not precond: 

256 if label: 

257 style = styles['no_precond'] 

258 else: 

259 style = styles['no_label'] 

260 else: 

261 style = next_style(styles['rotation']) 

262 style['label'] = label 

263 return style 

264 

265 def render_logics(self): 

266 # render logic definitions 

267 

268 for name, functions in states.items(): 

269 new_node(name, functions) 

270 

271 for source, info in transitions.items(): 

272 new_node(source, ) # assure node (with prefix) exists 

273 

274 s = f"{prefix}_{source}" 

275 for event, trxs in info.items(): 

276 for trx in trxs: 

277 target, precond, functions = z = trx[:3] 

278 

279 for zz in self.transitions.get(source, dict()).get(event, list()): 

280 if zz[:2] == z[:2]: 

281 break 

282 else: 

283 continue 

284 

285 new_node(target, ) # assure node (with prefix) exists 

286 

287 t = f"{prefix}_{target}" 

288 attrs = edge_attrs(source, event, target, precond, functions) 

289 if source == target: 

290 aux_node = new_aux_node(styles['invisible']) 

291 dg.edge(s, aux_node, **attrs) 

292 attrs.pop('label', None) 

293 dg.edge(aux_node, t, '', **attrs) 

294 foo = 1 

295 else: 

296 dg.edge(s, t, **attrs) 

297 

298 return dg 

299 

300 # Default Graph configuration 

301 if graph_cfg is None: 

302 graph_cfg = dict( 

303 graph_attr=dict( 

304 # splines="line", 

305 splnes="compound", 

306 model="subset", 

307 # model="circuit", 

308 # ranksep="1", 

309 # model="10", 

310 # mode="KK", # gradient descend 

311 mindist="2.5", 

312 ), 

313 edge_attr=dict( 

314 # len="1", 

315 # ranksep="1", 

316 ), 

317 # engine='sfdp', 

318 # engine='neato', 

319 # engine='dot', 

320 # engine='twopi', 

321 # engine='circo', 

322 engine='dot', 

323 ) 

324 

325 # Styles 

326 if styles is None: 

327 styles = dict() 

328 

329 for key, st in DEFAULT_STYLES.items(): 

330 styles.setdefault(key, st) 

331 

332 dg = graph = graphviz.Digraph(name=name, **graph_cfg) 

333 prefix = f"{self.__class__.__name__}" 

334 graph.body.append(f'\tlabel="Layer: {prefix}"\n') 

335 

336 for logic, states, transitions, mode, _ in self._get_layer_setups(): 

337 if logic in skip: 

338 continue 

339 

340 if include and logic not in include: 

341 continue 

342 

343 if isolated: 

344 prefix = f"{self.__class__.__name__}_{logic}" 

345 dg = graphviz.Digraph(name=f"cluster_{logic}", **graph_cfg) 

346 

347 render_logics(self) 

348 

349 if isolated: 

350 dg.body.append(f'\tlabel="Logic: {logic}"') 

351 graph.subgraph(dg) 

352 

353 graph.rendered = graph.render(filename=name, 

354 directory=path, format=format, view=view, cleanup=True) 

355 

356 return graph 

357 

358 

359 

360 

361 # ----------------------------------------------- 

362 # Layer definitions 

363 # ----------------------------------------------- 

364 def _get_layer_setups(self, include=None, skip=None): 

365 include = include or [] 

366 skip = skip or [] 

367 

368 match = re.compile('_setup_(?P<name>.*)').match 

369 names = [name for name in dir(self) if match(name)] 

370 names.sort() 

371 for name in names: 

372 logic = match(name).groupdict()['name'] 

373 

374 if logic in skip: 

375 continue 

376 if include and logic not in include: 

377 continue 

378 

379 func = getattr(self, name) 

380 states, transitions, mode = _call(func, **self.context) 

381 yield logic, states, transitions, mode, func.__doc__ 

382 

383 def _setup_term(self): 

384 """Set TERM and QUIT logic for the base layer.""" 

385 states = { 

386 STATE_INIT: [[], [], ['start']], 

387 STATE_READY: [[], [], []], 

388 STATE_END: [[], ['bye'], []], 

389 } 

390 transitions = { 

391 STATE_INIT: { 

392 None: [ 

393 [STATE_READY, [], []], 

394 ], 

395 }, 

396 STATE_READY: { 

397 EVENT_TERM: [ 

398 [STATE_READY, [], ['term']], 

399 ], 

400 EVENT_QUIT: [ 

401 [STATE_END, [], ['quit']], 

402 ], 

403 }, 

404 STATE_END: { 

405 }, 

406 } 

407 return states, transitions, MERGE_ADD 

408 

409 

410class STM(object): 

411 """Gather many Layers that share context and may receive same events 

412 but has independent internal states. 

413  

414 Provide a universal identifier that group all layers together. 

415  

416 Layer es la unidad mínima para poder atener eventos 

417 

418 STM es una superposición de Layers que comparten el mismo contexto de ejecución agrupados por un mismo identificador único. 

419  

420 - [ ] Cada STM sólo debe acceder a sus propios datos internos para que la lógica sea "auto-contenida" 

421 - [ ] El criterio de diseño es que dos threads no pueden invocar a la misma STM a la vez, por eso serían thread-safe si la lógica no accede a datos externos a la STM. 

422 - [ ] Prohibido usar Locks. Usar un evento para acceder a un recurso compartido por otras STM en vez de Locks. 

423 - [ ] STM tiene que proporcionar un método para salvar y recuperar su estado como (``__setstate__, __getstate__``) 

424 

425 """ 

426 

427class Transport(object): 

428 """TODO: different transports: sock, pipes, subprocess, etc.""" 

429 def __init__(self, url): 

430 self.url = url 

431 

432 def write(self, data, **info): 

433 """Write some data bytes to the transport. 

434 

435 This does not block; it buffers the data and arranges for it 

436 to be sent out asynchronously. 

437 """ 

438 raise NotImplementedError 

439 

440 def writelines(self, list_of_data): 

441 """Write a list (or any iterable) of data bytes to the transport. 

442 

443 The default implementation concatenates the arguments and 

444 calls write() on the result. 

445 """ 

446 data = b''.join(list_of_data) 

447 self.write(data) 

448 

449 def write_eof(self): 

450 """Close the write end after flushing buffered data. 

451 

452 (This is like typing ^D into a UNIX program reading from stdin.) 

453 

454 Data may still be received. 

455 """ 

456 raise NotImplementedError 

457 

458 def can_write_eof(self): 

459 """Return True if this transport supports write_eof(), False if not.""" 

460 raise NotImplementedError 

461 

462 def abort(self): 

463 """Close the transport immediately. 

464 

465 Buffered data will be lost. No more data will be received. 

466 The protocol's connection_lost() method will (eventually) be 

467 called with None as its argument. 

468 """ 

469 raise NotImplementedError 

470 

471 def __str__(self): 

472 return f"{self.__class__.__name__}: {self.url}" 

473 

474 def __repr__(self): 

475 return str(self) 

476 

477class SockTransport(Transport): 

478 def __init__(self, url, sock): 

479 super().__init__(url) 

480 self.sock = sock 

481 

482 def write(self, data, **info): 

483 # TODO: timeit: convert only when is not byte or always 

484 self.sock.send(data) 

485 

486 

487 

488class Protocol(object): 

489 """ 

490 Ref: https://docs.python.org/3/library/asyncio-protocol.html#asyncio-protocol 

491 """ 

492 def __init__(self, reactor, layer): 

493 self.transport = None 

494 self.reactor = reactor 

495 self.layer = layer 

496 

497 def connection_made(self, transport): 

498 self.transport = transport 

499 

500 def connection_lost(self, exc): 

501 """Called when the connection is lost or closed. 

502 

503 The argument is an exception object or None (the latter 

504 meaning a regular EOF is received or the connection was 

505 aborted or closed). 

506 """ 

507 self.transport = None 

508 

509 def data_received(self, data): 

510 """Called when a data is received from transport""" 

511 

512 def eof_received(self): 

513 """Called when a send or receive operation raises an OSError.""" 

514 

515class EchoProtocol(Protocol): 

516 """A simple ECHO demo protocol""" 

517 

518 def data_received(self, data): 

519 self.transport.write(data) 

520 

521class _TestBrowserProtcol(Protocol): 

522 def connection_made(self, transport): 

523 super().connection_made(transport) 

524 request = """GET / HTTP/1.1 

525Accept-Encoding: identity 

526Host: www.debian.org 

527User-Agent: Python-urllib/3.7 

528Connection: close 

529 

530""" 

531 request = request.replace('\n', '\r\n') 

532 request = bytes(request, 'utf-8') 

533 self.transport.write(request) 

534 

535 def data_received(self, data): 

536 """Called when a data is received from transport""" 

537 assert b'The document has moved' in data 

538 for line in str(data, 'utf-8').splitlines(): 

539 print(line) 

540 

541 # import urllib.request 

542 # import urllib.parse 

543 

544 # url = 'http://www.debian.org' 

545 # f = urllib.request.urlopen(url) 

546 # print(f.read().decode('utf-8')) 

547 

548 def eof_received(self): 

549 self.reactor.stop() 

550 

551 

552class Reactor(Recorder): 

553 """Holds several STM and provide a mechanism for STM to 

554 receive and submit events: 

555 

556 - use pub/sub paradigm. 

557 - use select() for channel monitoring and timers 

558 

559 Is focused on speed: 

560 - Layers define events using regexp. 

561 - Reactor use a cache of (event, state) updated in runtime. 

562 - Only events that may be attended in the next cycle are present in the cache. 

563 - Every Layer transition update the cache to reflect the events that  

564 this Layer can attend. 

565  

566  

567 - [ ] Es quien gestiona los canales y timers mediante "select". 

568 - [ ] Es quien instancia las STM y esta a su vez los Layers que lo componen. 

569 - [ ] Da soporte a persistencia para poder levantar y dormir un sistema. 

570 - [ ] Ofrece el tiempo universal de la máquina en microsegundos. Parchear ``time.time()`` 

571 - [ ] Ofrece servicios de configuración y almacenamientos de parámetros para cada STM individual. 

572 - [ ] Permite levantar o apagar las STM según fichero de configuración. 

573 - [ ] Monitoriza ese fichero de configuración para en caliente reflejar el estado de la máquina (como logger). 

574 - [ ] Puede separar distintos grupos de canales para que sean atendidos en threads independientes. 

575 - [ ] Si fuera necesario podrían desdoblarse en procesos independientes que se comunican entre si para garantizar un aislamiento total entre las librerias que se estén usando. 

576 - [ ] Graba eventos recibidos en ficheros comprimidos 'xz' para posterior repetición y depuración. 

577 - [ ] Los ficheros que levantan los Reactors pueden ser considerados como una sesión. 

578 - [ ] La sesión se puede definir mediante python, yaml, etc. Al final genera un diccionario que es el que se analizan sus cambios. 

579 - [ ] Un fichero python siempre será más potente que un simple yaml. Aunque sólo defina un diccionario, siempre podrá incluir librerias, hacer bucles, etc, similar a Sphinx-doc.  

580 - [ ] Usar logger para depurar 

581  

582 """ 

583 

584 def __init__(self, *args, **kwargs): 

585 super().__init__(*args, **kwargs) 

586 self.layers = WeakKeyDictionary() 

587 self.events = dict() 

588 self._queue = list() 

589 self._timers = list() 

590 self._transports = dict() 

591 self._protocols = dict() 

592 self._layer_transports = dict() 

593 

594 self.running = False 

595 

596 def attach(self, layer): 

597 """Attach a Layer to infrastructure: 

598 - evaluate all (event, state) pairs that the layer can process 

599 - add to reactor infrastructure 

600 """ 

601 layer._compile() 

602 # prepare event placeholders 

603 for state, event, trx in layer._trx_iterator(): 

604 self.events.setdefault(event, dict()) 

605 if isinstance(event, str): 

606 # prepare timers as well 

607 timer = event.split('each:') 

608 timer = timer[1:] 

609 if timer: 

610 timer = timer[0].split(',') 

611 timer.append(0) 

612 timer = [int(x) for x in timer[:2]] 

613 timer.reverse() 

614 timer.append(event) 

615 self._timers.append(timer) 

616 self._timers.sort() 

617 foo = 1 

618 

619 # set the initial state for the layer 

620 new_state = layer.state = STATE_INIT 

621 for ev, trx in layer.transitions[new_state].items(): 

622 self.events[ev][layer] = trx 

623 

624 # asure thet the context contains all public layer elements 

625 # so is not dependant on superclasses __init__() order 

626 context = dict([(k, getattr(layer, k)) for k in [k for k in dir(layer) if k[0] != '_']]) 

627 layer.context.update(context) 

628 

629 layer.reactor, self.layers[layer] = self, layer 

630 

631 def detach(self, layer): 

632 """Detach a Layer from infrastructure: 

633 - evaluate all (event, state) pairs that the layer can process 

634 - remove from reactor infrastructure""" 

635 for source, event, transitions in layer._trx_iterator(): 

636 info = self.events.get(event, {}) 

637 info.pop(layer, None) 

638 

639 # remove non persistent trsnsports created by layer 

640 for transport in self._layer_transports.pop(layer, []): 

641 self.close_channel(transport) 

642 

643 # remove layer and check is reactor must be running 

644 self.layers.pop(layer) 

645 self.running = len(self.layers) > 0 

646 

647 def publish(self, key, data=None): 

648 self._queue.append((key, data)) 

649 

650 def create_channel(self, protocol_factory, url, 

651 sock=None, local_addr=None, layer=None, 

652 **kw): 

653 """Create a transport connection linked with protocol instance from 

654 protocol factory. 

655 

656 - layer != None : transport is removed when layers detach 

657 - layer = None : transport is kept alive until reactor ends 

658 """ 

659 if sock is None: 

660 for family, type_, proto, raddr, laddr in self._analyze_url(url): 

661 sock = socket.socket(family=family, type=type_, proto=proto) 

662 # sock.setblocking(False) 

663 ok = True 

664 for addr in laddr: 

665 try: 

666 sock.bind(addr) 

667 break 

668 except OSError as why: 

669 ok = False 

670 if not ok: 

671 continue 

672 for addr in raddr: 

673 try: 

674 sock.connect(addr) 

675 break 

676 except OSError as why: 

677 ok = False 

678 if ok: 

679 break 

680 else: 

681 raise RuntimeError(f"Unable to create a sock for {url}") 

682 

683 protocol = protocol_factory(reactor=self, layer=layer) 

684 # TODO: different transport types based on url 

685 transport = SockTransport(url, sock) 

686 

687 self._transports[sock] = transport 

688 self._protocols[sock] = protocol 

689 protocol.connection_made(transport) 

690 

691 # store transport by layer. None means is persistent 

692 self._layer_transports.setdefault(layer, list()).append(transport) 

693 

694 return transport, protocol 

695 

696 def close_channel(self, transport=None, protocol=None): 

697 transport = transport or protocol.transport 

698 sock = transport.sock 

699 self._transports.pop(sock) 

700 self._protocols.pop(sock) 

701 sock.close() 

702 

703 def run(self): 

704 # print("> Starting Reactor loop") 

705 

706 self.t0 = time() 

707 self.running = True 

708 

709 while self.running: 

710 # try to evolute layers that do not require any events (None) 

711 events = self.events.get(None) 

712 if events: 

713 key = data = None 

714 else: 

715 # or wait for an external event 

716 key, data = self.next_event() # blocking 

717 events = self.events.get(key) 

718 if not events: 

719 continue # No Layer can attend this event, get the next one 

720 

721 # process all available transitions 

722 for layer, transitions in chain(list(events.items())): 

723 # check if a transition can be trigered 

724 ctx = layer.context 

725 ctx['key'] = key 

726 

727 # pass data into context 

728 # 1st in a isolated manner 

729 ctx['data'] = data 

730 

731 # 2nd directly, to allow callbacks receive the params directly 

732 # this may override some context variable 

733 # TODO: update the other way arround, but is slower 

734 isinstance(data, dict) and ctx.update(data) 

735 

736 for (new_state, preconds, funcs, comp_preconds) in transitions: 

737 try: 

738 # DO NOT check preconditions, it's makes slower 

739 for pre in comp_preconds: 

740 if not eval(pre, ctx): 

741 break 

742 else: 

743 if new_state != layer.state: # fast self-transitions 

744 # remove current state events 

745 for ev in layer.transitions[layer.state]: 

746 self.events[ev].pop(layer) # must exists!! 

747 # add new state events 

748 for ev, trx in layer.transitions[new_state].items(): 

749 self.events[ev][layer] = trx 

750 

751 # execute EXIT state functions (old state) 

752 for func in layer.states[layer.state][GROUP_EXIT]: 

753 # await func(**ctx) if func.__code__.co_flags & CO_COROUTINE else func(**ctx) 

754 func(**ctx) 

755 

756 # execute transition functions 

757 for func in funcs: 

758 # await func(**ctx) if func.__code__.co_flags & CO_COROUTINE else func(**ctx) 

759 func(**ctx) 

760 

761 layer.state = new_state 

762 

763 # execute ENTRY state functions (new state) 

764 for func in layer.states[new_state][GROUP_ENTRY]: 

765 # asyncio.iscoroutine(func) 

766 # func(**ctx) if func.__code__.co_flags & CO_COROUTINE else func(**ctx) 

767 func(**ctx) 

768 

769 # execute DO state functions (new state) 

770 for func in layer.states[new_state][GROUP_DO]: 

771 # asyncio.iscoroutine(func) 

772 # await func(**ctx) if func.__code__.co_flags & CO_COROUTINE else func(**ctx) 

773 func(**ctx) 

774 

775 break # assume there's only 1 transition possible each time 

776 except Exception as why: 

777 print() 

778 print(f"- Reactor {'-'*70}") 

779 print(f"*** ERROR: {why} ***") 

780 traceback.print_exc() 

781 print("-" * 80) 

782 foo = 1 

783 

784 # close all remaining transports 

785 for transport in list(self._transports.values()): 

786 print(f" - closing: {transport}") 

787 self.close_channel(transport) 

788 

789 # print("< Exiting Reactor loop") 

790 foo = 1 

791 

792 def stop(self): 

793 self.publish(EVENT_TERM) 

794 self.running = False 

795 

796 @property 

797 def time(self): 

798 return time() - self.t0 

799 

800 def next_event(self): 

801 """Blocks waiting for an I/O event or timer. 

802 Try to compensate delays by substracting time.time() and sub operations.""" 

803 def close_sock(fd): 

804 self._protocols[fd].eof_received() 

805 self.close_channel(self._transports[fd]) 

806 

807 while True: 

808 if self._queue: 

809 return self._queue.pop(0) 

810 

811 # there is not events to process. look for timers and I/O 

812 if self._timers: 

813 when, restart, key = timer = self._timers[0] 

814 seconds = when - self.time 

815 if seconds > 0: 

816 rx, _, ex = select(self._transports, [], self._transports, seconds) 

817 for fd in rx: 

818 try: 

819 raw = fd.recv(0xFFFF) 

820 if raw: 

821 self._protocols[fd].data_received(raw) 

822 else: 

823 close_sock(fd) 

824 except Exception as why: 

825 close_sock(fd) 

826 pass 

827 

828 for fd in ex: # TODO: remove if never is invoked 

829 close_sock(fd) 

830 foo = 1 

831 else: 

832 self.publish(key, None) 

833 # -------------------------- 

834 # rearm timer 

835 self._timers.pop(0) 

836 if restart <= 0: 

837 continue # is a timeout timer, don't restart it 

838 when += restart 

839 timer[0] = when 

840 # insert in right position 

841 i = 0 

842 while i < len(self._timers): 

843 if self._timers[i][0] > when: 

844 self._timers.insert(i, timer) 

845 break 

846 i += 1 

847 else: 

848 self._timers.append(timer) 

849 else: 

850 # duplicate code for faster execution 

851 rx, _, _ = select(self._transports, [], [], 1) 

852 for fd in rx: 

853 try: 

854 raw = fd.recv(0xFFFF) 

855 if raw: 

856 self._protocols[fd].data_received(raw) 

857 else: 

858 close_sock(fd) 

859 except Exception as why: 

860 close_sock(fd) 

861 pass 

862 foo = 1 

863 foo = 1 

864 

865 def graph(self, dg=None, graph_cfg=None, styles=None, view=True, format='svg'): 

866 if graph_cfg is None: 

867 graph_cfg = dict( 

868 graph_attr=dict( 

869 # splines="line", 

870 # splnes="compound", 

871 # model="subset", 

872 # model="circuit", 

873 ranksep="1", 

874 model="10", 

875 mode="KK", # gradient descend 

876 mindist="2.5", 

877 ), 

878 edge_attr=dict( 

879 len="5", 

880 ranksep="3", 

881 ), 

882 # engine='sfdp', 

883 # engine='neato', 

884 # engine='dot', 

885 # engine='twopi', 

886 # engine='circo', 

887 engine='dot', 

888 ) 

889 

890 # Styles 

891 if styles is None: 

892 styles = dict() 

893 

894 for name, st in DEFAULT_STYLES.items(): 

895 styles.setdefault(name, st) 

896 

897 # Create a new DG or use the expernal one 

898 if dg is None: 

899 dg = graphviz.Digraph(**graph_cfg) 

900 

901 for layer in self.layers: 

902 graph = layer.graph(graph_cfg, styles, format=format) 

903 name = layer.__class__.__name__ 

904 graph.body.append(f'\tlabel="Layer: {name}"\n') 

905 dg.subgraph(graph) 

906 

907 # dg._engine = 'neato' 

908 dg.render('reactor.dot', format='svg', view=view) 

909 foo = 1 

910 

911 

912 def _analyze_url(self, url): 

913 url_ = parse_uri(url) 

914 # family, type_, proto, raddr, laddr 

915 family, type_, proto, port = { 

916 'tcp': (socket.AF_INET, socket.SOCK_STREAM, -1, None), 

917 'http': (socket.AF_INET, socket.SOCK_STREAM, -1, 80), 

918 'udp': (socket.AF_INET, socket.SOCK_DGRAM, -1, None), 

919 }.get(url_['fscheme']) 

920 

921 raddr = [(url_['host'], url_['port'] or port)] 

922 laddr = [] 

923 

924 yield family, type_, proto, raddr, laddr 

925 

926 

927 

928 

929 

930 

931class DebugReactor(Reactor): 

932 """Reactor with debugging and Statisctical information""" 

933 def __init__(self): 

934 super().__init__() 

935 self.__stats = dict() 

936 for k in ('cycles', 'publish', 'max_queue', 'max_channels', ): 

937 self.__stats[k] = 0 

938 

939 def publish(self, key, data=None): 

940 Reactor.publish(self, key, data) 

941 self.__stats['publish'] += 1 

942 self.__stats['max_queue'] = max([len(self._queue), \ 

943 self.__stats['max_queue']]) 

944 

945 def create_channel(self, url, **kw): 

946 Reactor.publish(self, url=url, **kw) 

947 

948 s = self.__stats.setdefault('create_channel', dict()) 

949 s[url] = s.get(url, 0) + 1 

950 

951 self.__stats['max_channels'] = max([len(self._transports), \ 

952 self.__stats['max_channels']]) 

953 

954 def run(self): 

955 # print("> Starting Reactor loop") 

956 s = self.__stats 

957 

958 # events received 

959 evs = s['events'] = dict() 

960 for ev in self.events: 

961 evs[ev] = 0 

962 evs[None] = 0 

963 

964 # states 

965 sts = s['states'] = dict() 

966 trx = s['transitions'] = dict() 

967 trx_f = s['transitions_failed'] = dict() 

968 for layer in self.layers: 

969 sts_ = sts[layer] = dict() 

970 trx_ = trx[layer] = dict() 

971 for name, states, transitions, mode, _ in layer._get_layer_setups(): 

972 for state in states: 

973 sts_[state] = 0 

974 for tx in transitions: 

975 trx_[tx[0]] = 0 

976 trx_f[tx[0]] = 0 

977 

978 # the same code as Reactor.run() but updating stats 

979 # NOTE: we need to update the code manually if base class changes 

980 

981 self.t0 = time() 

982 self.running = True 

983 

984 while self.running: 

985 # try to evolute layers that do not require any events (None) 

986 events = self.events.get(None) 

987 if events: 

988 key = data = None 

989 else: 

990 # or wait for an external event 

991 key, data = self.next_event() # blocking 

992 events = self.events.get(key) 

993 if not events: 

994 continue 

995 

996 s['cycles'] += 1 # << 

997 evs[key] += 1 # << 

998 

999 # process all available transitions 

1000 for layer, transitions in chain(list(events.items())): 

1001 # check if a transition can be trigered 

1002 ctx = layer.context 

1003 ctx['key'] = key 

1004 

1005 # pass data into context 

1006 # 1st in a isolated manner 

1007 ctx['data'] = data 

1008 

1009 # 2nd directly, to allow callbacks receive the params directly 

1010 # this may override some context variable 

1011 # TODO: update the other way arround, but is slower 

1012 isinstance(data, dict) and ctx.update(data) 

1013 

1014 for (new_state, preconds, funcs, comp_preconds) in transitions: 

1015 try: 

1016 # DO NOT check preconditions, it's makes slower 

1017 for pre in comp_preconds: 

1018 if not eval(pre, ctx): 

1019 trx_f[new_state] += 1 # << 

1020 break 

1021 else: 

1022 trx_[new_state] += 1 # << 

1023 if new_state != layer.state: # fast self-transitions 

1024 # remove current state events 

1025 for ev in layer.transitions[layer.state]: 

1026 self.events[ev].pop(layer) # must exists!! 

1027 # add new state events 

1028 for ev, trx in layer.transitions[new_state].items(): 

1029 self.events[ev][layer] = trx 

1030 

1031 # execute EXIT state functions (old state) 

1032 for func in layer.states[layer.state][GROUP_EXIT]: 

1033 # await func(**ctx) if func.__code__.co_flags & CO_COROUTINE else func(**ctx) 

1034 func(**ctx) 

1035 

1036 # execute transition functions 

1037 for func in funcs: 

1038 # await func(**ctx) if func.__code__.co_flags & CO_COROUTINE else func(**ctx) 

1039 func(**ctx) 

1040 

1041 layer.state = new_state 

1042 

1043 sts[layer][new_state] += 1 # << 

1044 

1045 # execute ENTRY state functions (new state) 

1046 for func in layer.states[new_state][GROUP_ENTRY]: 

1047 # asyncio.iscoroutine(func) 

1048 # func(**ctx) if func.__code__.co_flags & CO_COROUTINE else func(**ctx) 

1049 func(**ctx) 

1050 

1051 # execute DO state functions (new state) 

1052 for func in layer.states[new_state][GROUP_DO]: 

1053 # asyncio.iscoroutine(func) 

1054 # await func(**ctx) if func.__code__.co_flags & CO_COROUTINE else func(**ctx) 

1055 func(**ctx) 

1056 

1057 break # assume there's only 1 transition possible each time 

1058 except Exception as why: 

1059 print() 

1060 print(f"- Reactor {'-'*70}") 

1061 print(f"*** ERROR: {why} ***") 

1062 traceback.print_exc() 

1063 print("-" * 80) 

1064 foo = 1 

1065 

1066 # close all remaining transports 

1067 for transport in list(self._transports.values()): 

1068 print(f" - closing: {transport}") 

1069 self.close_channel(transport) 

1070 

1071 # print("< Exiting Reactor loop") 

1072 foo = 1 

1073 

1074class DocRender(object): 

1075 def __init__(self, reactor): 

1076 self.reactor = reactor 

1077 self.env = Environment( 

1078 loader=PackageLoader(self.__module__, 'jinja2'), 

1079 autoescape=select_autoescape(['html', 'xml']) 

1080 ) 

1081 

1082 def render(self, root, include=None, skip=['term']): 

1083 """Render the documentation in Markdown formar ready to be used 

1084 by Hugo static html generator. 

1085 

1086 - main file is 'stm.md' 

1087 - graphs are created in SVG format under stm/xxx.svg directory (hugo compatible) 

1088 

1089 """ 

1090 include = include or [] 

1091 skip = skip or [] 

1092 

1093 ctx = dict(format='svg', path=os.path.join(root, 'stm'), skip=skip) 

1094 

1095 # render main Markdown file 

1096 template = self.env.get_template('layer.md') 

1097 ctx['layers'] = self.reactor.layers 

1098 with open(os.path.join(root, 'stm.md'), 'w') as f: 

1099 out = template.render(**ctx) 

1100 f.write(out) 

1101 

1102 # render each layer logic in a single diagram 

1103 for layer in self.reactor.layers: 

1104 ctx['layer'] = layer 

1105 ctx['layer_name'] = layer.__class__.__name__ 

1106 ctx['include'] = [] 

1107 ctx['name'] = f"{ctx['layer_name']}" 

1108 ctx['graph'] = graph = _call(layer.graph, **ctx) 

1109 

1110 for logic, states, transitions, doc in layer._get_layer_setups(include, skip): 

1111 ctx['include'] = [logic] 

1112 ctx['name'] = f"{ctx['layer_name']}_{logic}" 

1113 ctx['graph'] = graph = _call(layer.graph, **ctx) 

1114 

1115 

1116 

1117 

1118 

1119 for logic, states, transitions in self._get_layer_setups(): 

1120 if logic in skip: 

1121 continue 

1122 

1123 if isolated: 

1124 prefix = f"{self.__class__.__name__}_{logic}" 

1125 dg = graphviz.Digraph(name=f"cluster_{logic}", **graph_cfg) 

1126 

1127 render_logics() 

1128 if isolated: 

1129 dg.body.append(f'\tlabel="Logic: {logic}"') 

1130 graph.subgraph(dg) 

1131 

1132