Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import pytest 

2 

3import random 

4import operator 

5from time import time, sleep 

6from select import select 

7 

8from gutools.tools import get_calling_function, flatten 

9from gutools.utests import speed_meter 

10from gutools.stm import Layer, Reactor, DebugReactor, STATE_INIT, STATE_READY, STATE_END, \ 

11 QUITE_STATE, EVENT_QUIT, EVENT_TERM, MERGE_ADD, MERGE_REPLACE_EXISTING 

12 

13from gutools.stm import _TestBrowserProtcol 

14 

15class _TestReactor(Reactor): 

16 def __init__(self, events=None): 

17 super().__init__() 

18 self._fake_events = events # iterator 

19 self._queue = list() 

20 

21 def stop(self): 

22 print("Stoping reactor") 

23 super().stop() 

24 

25 # def next_event(self): 

26 # try: 

27 # while True: 

28 # if self._queue: 

29 # return self._queue.pop(0) 

30 # elif self._fake_events: 

31 # return self._fake_events.__next__() 

32 # sleep(0.1) 

33 # except StopIteration: 

34 # self.running = False 

35 # return EVENT_TERM, None 

36 

37 def next_event(self): 

38 """Blocks waiting for an I/O event or timer. 

39 Try to compensate delays by substracting time.time() and sub operations.""" 

40 def close_sock(fd): 

41 self._protocols[fd].eof_received() 

42 self.close_channel(self._transports[fd]) 

43 

44 while True: 

45 if self._queue: 

46 return self._queue.pop(0) 

47 elif self._fake_events: 

48 try: 

49 return self._fake_events.__next__() 

50 except StopIteration: 

51 self.running = False 

52 return EVENT_TERM, None 

53 

54 # there is not events to process. look for timers and I/O 

55 if self._timers: 

56 when, restart, key = timer = self._timers[0] 

57 seconds = when - self.time 

58 if seconds > 0: 

59 rx, _, ex = select(self._transports, [], self._transports, seconds) 

60 for fd in rx: 

61 try: 

62 raw = fd.recv(0xFFFF) 

63 if raw: 

64 self._protocols[fd].data_received(raw) 

65 else: 

66 close_sock(fd) 

67 except Exception as why: 

68 close_sock(fd) 

69 pass 

70 

71 for fd in ex: # TODO: remove if never is invoked 

72 close_sock(fd) 

73 foo = 1 

74 else: 

75 self.publish(key, None) 

76 # -------------------------- 

77 # rearm timer 

78 self._timers.pop(0) 

79 if restart <= 0: 

80 continue # is a timeout timer, don't restart it 

81 when += restart 

82 timer[0] = when 

83 # insert in right position 

84 i = 0 

85 while i < len(self._timers): 

86 if self._timers[i][0] > when: 

87 self._timers.insert(i, timer) 

88 break 

89 i += 1 

90 else: 

91 self._timers.append(timer) 

92 else: 

93 # duplicate code for faster execution 

94 rx, _, _ = select(self._transports, [], [], 1) 

95 for fd in rx: 

96 try: 

97 raw = fd.recv(0xFFFF) 

98 if raw: 

99 self._protocols[fd].data_received(raw) 

100 else: 

101 close_sock(fd) 

102 except Exception as why: 

103 close_sock(fd) 

104 pass 

105 foo = 1 

106 foo = 1 

107 

108def populate1(container, population, n): 

109 while n: 

110 ctx = container 

111 for x in population: 

112 key = str(random.randint(0, x)) 

113 ctx = ctx.setdefault(key, dict()) 

114 

115 key = str(random.randint(0, population[-1])) 

116 ctx = ctx.setdefault(key, list()) 

117 ctx.append(n) 

118 n -= 1 

119 

120def populate2(container, population, n): 

121 while n: 

122 key = tuple([str(random.randint(0, x)) for x in population]) 

123 container[key] = n 

124 n -= 1 

125 

126class _TestLayer(Layer): 

127 

128 def _setup__testlayer(self): 

129 states = { 

130 STATE_END: [[], ['bye'], []], 

131 } 

132 

133 transitions = { 

134 STATE_READY: { 

135 'each:5,1': [ 

136 ['READY', [], ['timer']], 

137 ], 

138 'each:5,21': [ 

139 [STATE_READY, [], ['bye']], 

140 ], 

141 }, 

142 } 

143 return states, transitions, MERGE_ADD 

144 

145 def start(self, **kw): 

146 print("Hello World!!") 

147 self.t0 = time() 

148 self._func_calls = dict() 

149 self._log_function() 

150 

151 def term(self, key, **kw): 

152 elapsed = time() - self.t0 

153 print(f"Term {key}: {elapsed}") 

154 super().term(key, **kw) 

155 

156 def timer(self, **kw): 

157 "Empty timer to be overrided" 

158 

159 def _log_function(self): 

160 func = get_calling_function(level=2) 

161 name = func.__func__.__name__ 

162 self._func_calls.setdefault(name, 0) 

163 self._func_calls[name] += 1 

164 

165 def _check_log(self, expected): 

166 """Check if observerd calls match expected ones. 

167 Expected values can be integer and iterables. 

168 Ranges may be defined with strings like '7-8' 

169 """ 

170 for name, _ve in expected.items(): 

171 ve = set() 

172 for v in flatten(list([_ve])): 

173 # allow ranges 

174 v = str(v).split('-') 

175 v.append(v[0]) 

176 ve.update(range(int(v[0]), int(v[1]) + 1)) 

177 

178 vr = self._func_calls.get(name, None) 

179 if vr not in ve: 

180 self.reactor.stop() 

181 raise RuntimeError(f"Fuction {name} is expected to be called {ve} time, but where {vr}") 

182 

183 

184class Foo(_TestLayer): 

185 def __init__(self, states=None, transitions=None, context=None): 

186 super().__init__(states, transitions, context) 

187 

188 self.b = 0 

189 

190 def _setup_test_foo_1(self): 

191 states = { 

192 STATE_INIT: [[], [], ['hello']], 

193 STATE_READY: [[], ['ready'], []], 

194 STATE_END: [[], ['stop'], []], 

195 } 

196 transitions = { 

197 } 

198 return states, transitions, MERGE_ADD 

199 

200 def _setup_test_foo_2(self): 

201 states = { 

202 } 

203 transitions = { 

204 STATE_READY: { 

205 EVENT_TERM: [ 

206 [STATE_READY, ['b <= 5'], ['not_yet']], 

207 [STATE_READY, ['b > 5'], ['from_init_to_end', 'term']], 

208 ], 

209 }, 

210 } 

211 return states, transitions, MERGE_REPLACE_EXISTING 

212 

213 def timer(self, **kw): 

214 self.context['b'] += 1 

215 print(f">> timer: b={self.context['b']}") 

216 

217 def hello(self, **kw): 

218 print(">> Hello!") 

219 self._log_function() 

220 

221 def ready(self, **kw): 

222 print(">> Ready") 

223 self._log_function() 

224 

225 def stop(self, **kw): 

226 print(">> Stop !!") 

227 self._log_function() 

228 

229 def not_yet(self, key, **kw): 

230 print(f'>> Received: {key}, but not_yet()') 

231 self._log_function() 

232 

233 def from_init_to_end(self, key, **kw): 

234 print(f'>> Received: {key}, OK shutdown now') 

235 self._log_function() 

236 

237 def term(self, **kw): 

238 expected = { 

239 'from_init_to_end': 1, 

240 'hello': 1, 

241 'not_yet': 2, 

242 'ready': 12, 

243 'start': 1 

244 } 

245 self._check_log(expected) 

246 super().term(**kw) 

247 

248class iRPNCalc(Layer): 

249 operations = { 

250 '+': operator.add, 

251 '-': operator.sub, 

252 '*': operator.mul, 

253 '/': operator.truediv, 

254 'neg': operator.neg, 

255 } 

256 

257 def __init__(self, states, transitions): 

258 self.stack = [] 

259 super().__init__(states=states, transitions=transitions) 

260 foo = 1 

261 

262 def start(self, data, **kw): 

263 print("Hello World!!") 

264 

265 def push(self, data, **kw): 

266 self.stack.append(data) 

267 

268 def exec1(self, data, **kw): 

269 a = self.stack.pop() 

270 r = self.operations[data](a) 

271 self.stack.append(r) 

272 

273 def exec2(self, data, **kw): 

274 b = self.stack.pop() 

275 a = self.stack.pop() 

276 try: 

277 r = self.operations[data](a, b) 

278 self.stack.append(r) 

279 except ZeroDivisionError: 

280 pass 

281 

282 

283class RPNCalc1(iRPNCalc): 

284 def __init__(self): 

285 # Simplier RPN, less states 

286 states = { 

287 STATE_INIT: [[], ['start'], []], 

288 'READY': QUITE_STATE, 

289 'PUSH': [[], ['push'], []], 

290 'EXEC1': [[], ['exec1'], []], 

291 'EXEC2': [[], ['exec2'], []], 

292 STATE_END: [[], ['bye'], []], 

293 } 

294 

295 transitions = { 

296 'INIT': { 

297 None: [ 

298 ['READY', [], []], 

299 ], 

300 }, 

301 'READY': { 

302 'input': [ 

303 ['PUSH', ['isinstance(data, int)'], []], 

304 ['EXEC2', ["data in ('+', '-', '*', '/')", 'len(stack) >= 2'], []], 

305 ['EXEC1', ["data in ('neg', )", 'len(stack) >= 1'], []], 

306 ], 

307 EVENT_TERM: [ 

308 [STATE_END, [], []], 

309 ], 

310 }, 

311 'PUSH': { 

312 None: [ 

313 ['READY', [], []], 

314 ], 

315 }, 

316 'EXEC1': { 

317 None: [ 

318 ['READY', [], []], 

319 ], 

320 }, 

321 'EXEC2': { 

322 None: [ 

323 ['READY', [], []], 

324 ], 

325 }, 

326 } 

327 super().__init__(states, transitions) 

328 

329class RPNCalc2(iRPNCalc): 

330 def __init__(self): 

331 # Simplier RPN, less states 

332 states = { 

333 STATE_INIT: [[], ['start'], [], []], 

334 'READY': QUITE_STATE, 

335 STATE_END: [[], ['bye'], [], []], 

336 } 

337 

338 transitions = { 

339 'INIT': { 

340 None: [ 

341 ['READY', [], []], 

342 ], 

343 }, 

344 'READY': { 

345 'input': [ 

346 ['READY', ['isinstance(data, int)'], ['push']], 

347 ['READY', ["data in ('+', '-', '*', '/')", 'len(stack) >= 2'], ['exec2']], 

348 ['READY', ["data in ('neg', )", 'len(stack) >= 1'], ['exec1']], 

349 ], 

350 EVENT_TERM: [ 

351 [STATE_END, [], []], 

352 ], 

353 }, 

354 STATE_END: { 

355 }, 

356 } 

357 super().__init__(states, transitions) 

358 

359 

360 

361class Clock(_TestLayer): 

362 

363 def _setup_test_clock(self): 

364 states = { 

365 } 

366 transitions = { 

367 STATE_READY: { 

368 # set an additional timer 

369 'each:3,2': [ 

370 [STATE_READY, [], ['timer']], 

371 ], 

372 }, 

373 } 

374 return states, transitions, MERGE_ADD 

375 

376 def timer(self, key, **kw): 

377 elapsed = time() - self.t0 

378 print(f" > Timer {key}: {elapsed}") 

379 restart, offset = [int(x) for x in key[5:].split(',')] 

380 

381 cycles = (elapsed - offset) / restart 

382 assert cycles - int(cycles) < 0.01 

383 

384 

385class Parent(_TestLayer): 

386 

387 def _setup_test_parent(self): 

388 states = { 

389 'FOO': [[], ['bar'], []], 

390 } 

391 transitions = { 

392 } 

393 return states, transitions, MERGE_ADD 

394 

395 def bar(self, **kw): 

396 pass 

397 

398class Crawler(_TestLayer): 

399 """Dummy Web Crawler for testing dynamic channels creation and 

400 protocols. 

401 

402 - a timer creates a new channel from time to time. 

403 - a timer use a free channel to make a request. 

404 - a timeout will term the reactor. 

405 

406 """ 

407 def __init__(self, states=None, transitions=None, context=None): 

408 super().__init__(states, transitions, context) 

409 self.channels = list() # free channels 

410 self.working = list() # in-use channels 

411 self.downloaded = 0 

412 

413 def _setup_test_crawler(self): 

414 states = { 

415 'GET': [[], ['get_page'], []], 

416 'SAVE': [[], ['save_page'], []], 

417 } 

418 transitions = { 

419 STATE_READY: { 

420 'each:3,2': [ 

421 [STATE_READY, ['len(channels) <= 3'], ['new_channel']], 

422 ], 

423 'each:1,1': [ 

424 ['GET', ['len(working) <= 3 and downloaded < 10'], []], # just to use a state 

425 ], 

426 'each:2,1': [ 

427 ['SAVE', ['len(working) > 3'], []], # just to use a state 

428 ], 

429 'each:5,21': [ 

430 [STATE_READY, ['downloaded >= 10'], ['bye']], 

431 ], 

432 }, 

433 'GET': { 

434 None: [ 

435 [STATE_READY, [], []], # going back to READY 

436 ], 

437 }, 

438 'SAVE': { 

439 None: [ 

440 [STATE_READY, [], []], # going back to READY 

441 ], 

442 }, 

443 } 

444 return states, transitions, MERGE_REPLACE_EXISTING 

445 

446 def new_channel(self, **kw): 

447 print(' > new_channel') 

448 self._log_function() 

449 self.channels.append(False) 

450 

451 def get_page(self, **kw): 

452 "Simulate a download page request" 

453 for i, working in enumerate(self.channels): 

454 if not working: 

455 self._log_function() 

456 print(' > get_page') 

457 self.channels[i] = True 

458 self.working.append(i) 

459 break 

460 

461 def save_page(self, **kw): 

462 "Simulate that page has been downloaded and saved to disk" 

463 

464 for worker, channel in reversed(list(enumerate(self.working))): 

465 if random.randint(0, 100) < 25: 

466 self._log_function() 

467 self.context['downloaded'] += 1 

468 print(f" > save_page: {self.context['downloaded']}") 

469 self.channels[channel] = False 

470 self.working.pop(worker) 

471 

472 def term(self, **kw): 

473 expected = { 

474 'get_page': '12-13', 

475 'new_channel': 4, 

476 'save_page': '10-13', 

477 'start': 1 

478 } 

479 self._check_log(expected) 

480 super().term(**kw) 

481 

482 

483class HashableEvents(Layer): 

484 """Test class for testing events using any hashable event 

485 - strings 

486 - tuples 

487 - integers/floats 

488 """ 

489 def _setup_test_hashable_events(self): 

490 self.channels = list() # free channels 

491 self.working = list() # in-use channels 

492 

493 states = { 

494 } 

495 transitions = { 

496 STATE_READY: { 

497 ('hello', 'world'): [ 

498 [STATE_READY, [], ['hello_world']], 

499 ], 

500 1: [ 

501 [STATE_READY, [], ['uno']], 

502 ], 

503 }, 

504 } 

505 return states, transitions, MERGE_ADD 

506 

507 def hello_world(self, **kw): 

508 print(' > hello_world') 

509 foo = 1 

510 

511 def uno(self, **kw): 

512 print(' > uno') 

513 foo = 1 

514 

515 

516# ----------------------------------------------------- 

517# Atlas demo 

518# ----------------------------------------------------- 

519class GraphExample(_TestLayer): 

520 def _setup_test_graph(self): 

521 states = { 

522 'ASK': [[], ['bar'], []], 

523 'WAIT': [[], [], []], 

524 } 

525 transitions = { 

526 STATE_READY: { 

527 'each:5,1': [ 

528 ['ASK', [], ['bar']], 

529 ], 

530 }, 

531 'ASK': { 

532 None: [ 

533 ['WAIT', [], []], 

534 ], 

535 }, 

536 } 

537 return states, transitions, MERGE_ADD 

538 

539 def bar(self, **kw): 

540 pass 

541 

542 

543# ----------------------------------------------------- 

544# tests 

545# ----------------------------------------------------- 

546 

547def test_layer(): 

548 

549 foo = Foo() 

550 reactor = _TestReactor() 

551 reactor.attach(foo) 

552 # reactor.graph() 

553 reactor.run() 

554 foo = 1 

555 

556def test_timeit(): 

557 population = [100, 100, 10] 

558 

559 states1 = dict() 

560 populate1(states1, population, n=10000) 

561 

562 N = 1000000 

563 

564 i = 0 

565 key = [str(random.randint(0, x)) for x in population] 

566 t0 = time() 

567 while i < N: 

568 info = states1 

569 for k in key: 

570 info = info.get(k, {}) 

571 i += 1 

572 t1 = time() 

573 

574 states2 = dict() 

575 populate2(states2, population, n=10000) 

576 key = [str(random.randint(0, x)) for x in population] 

577 

578 i = 0 

579 key = [str(random.randint(0, x)) for x in population] 

580 info = states2 

581 t2 = time() 

582 while i < N: 

583 k = tuple(key) 

584 info = states2.get(k, {}) 

585 i += 1 

586 t3 = time() 

587 

588 import pandas as pd 

589 t4 = time() 

590 df_1 = pd.DataFrame() 

591 

592 speed1 = N / (t1 - t0) 

593 speed2 = N / (t3 - t2) 

594 

595 print(f"{speed1}") # around 28e6 loops/sec 

596 print(f"{speed2}") # around 40e6 loops/sec 

597 print(f"{speed2/speed1}") # aound 1.5 faster 

598 print(f"Pandas: {t4-t3}") 

599 

600def test_await_vs_normal(): 

601 N = 10000000 

602 

603 def foo1(): 

604 return 

605 

606 async def foo2(): 

607 return 

608 

609 def timeit0(i): 

610 t0 = time() 

611 while i > 0: 

612 foo1() 

613 i -= 1 

614 t1 = time() 

615 return t1 - t0 

616 

617 async def timeit2(i): 

618 t0 = time() 

619 while i > 0: 

620 await foo2() 

621 i -= 1 

622 t1 = time() 

623 return t1 - t0 

624 

625 e1 = timeit0(N) 

626 import asyncio 

627 e2 = asyncio.run(timeit2(N)) 

628 

629 speed1 = N / (e1) 

630 speed2 = N / (e2) 

631 

632 print(f"{speed1}") # around 11e6 loops/sec 

633 print(f"{speed2}") # around 5e6 loops/sec 

634 print(f"{speed1/speed2}") # aound x2 faster 

635 

636 

637def test_clock(): 

638 reactor = _TestReactor() 

639 clock = Clock() 

640 reactor.attach(clock) 

641 reactor.run() 

642 foo = 1 

643 

644def test_hierarchy(): 

645 parent = Parent() 

646 

647 assert parent.states == { 

648 'END': [[], ['bye'], []], 

649 'FOO': [[], ['bar'], []], 

650 'INIT': [[], [], ['start']], 

651 'READY': [[], [], []] 

652 } 

653 

654 assert parent.transitions == { 

655 'END': {}, 

656 'INIT': {None: [['READY', [], []]]}, 

657 'READY': { 

658 EVENT_QUIT: [['END', [], ['quit']]], 

659 EVENT_TERM: [['READY', [], ['term']]], 

660 'each:5,1': [['READY', [], ['timer']]], 

661 'each:5,21': [['READY', [], ['bye']]]}, 

662 } 

663 

664 foo = 1 

665 

666 

667def test_calc(): 

668 """Compare two different STM definition styles 

669  

670 RPNCalc2: loop over same state doing actions on transitions 

671 RPNCalc1: has more states and doing actions entering in state 

672  

673 Is supposed RPNCalc1 stylel should be faster than RPNCalc2 one 

674  

675 Finally, RPNCalc1 stats are saved on a csv file using speed_meter() helper 

676 for comparison when changing STM internal libray. 

677 """ 

678 

679 def monkey_calc(N): 

680 for i in range(N): 

681 r = random.randint(0, 8 + len(iRPNCalc.operations)) 

682 if r <= 9: 

683 yield 'input', r 

684 else: 

685 ops = list(iRPNCalc.operations.keys()) 

686 yield 'input', list(iRPNCalc.operations)[r - 9] 

687 

688 foo = 1 

689 

690 N = 500000 

691 

692 # Testing RPNCalc1 definition 

693 reactor = _TestReactor(monkey_calc(N)) 

694 calc = RPNCalc1() 

695 reactor.attach(calc) 

696 reactor.graph() 

697 

698 t0 = time() 

699 reactor.run() 

700 e1 = time() - t0 

701 speed1 = N / e1 

702 print(f'Speed: {speed1}') 

703 

704 # Just using speed_meter() helper for writedown 

705 # RPNCalc1 stats 

706 

707 setup = """# This code must be executed for each iteration 

708reactor = _TestReactor(monkey_calc(N)) 

709calc = RPNCalc1() 

710reactor.attach(calc) 

711""" 

712 env = globals() 

713 env.update(locals()) 

714 

715 test = dict( 

716 stmt='reactor.run()', 

717 setup=setup, 

718 globals=env, 

719 number=1, 

720 repeat=3, 

721 N=N, 

722 label='RPN1Calc speed' 

723 ) 

724 

725 r = speed_meter(**test) 

726 print(r['speed']) 

727 

728 # Testing RPNCalc2 definition 

729 reactor = _TestReactor(monkey_calc(N)) 

730 calc = RPNCalc2() 

731 reactor.attach(calc) 

732 reactor.graph() 

733 

734 t0 = time() 

735 reactor.run() 

736 e2 = time() - t0 

737 

738 speed2 = N / e2 

739 print(f'Speed: {speed2}') 

740 

741 print(f"RPNCalc2 is {speed2/speed1:.4} times faster than RPNCalc1") 

742 foo = 1 

743 

744 

745def test_graph_layer(): 

746 monitor = GraphExample() 

747 reactor = _TestReactor() 

748 reactor.attach(monitor) 

749 reactor.graph(view=False) 

750 foo = 1 

751 

752def test_create_channel(): 

753 reactor = Reactor() 

754 url = 'http://www.debian.org' 

755 

756 # create a protocol and transport without layer (persistent until reactor ends) 

757 proto, trx = reactor.create_channel(url, factory=_TestBrowserProtcol) 

758 

759 reactor.run() 

760 foo = 1 

761 

762def test_crawler(): 

763 reactor = Reactor() 

764 crawler = Crawler() 

765 reactor.attach(crawler) 

766 reactor.run() 

767 foo = 1 

768 

769def test_hashable_events(): 

770 def events(): 

771 yield ('hello', 'world'), None 

772 yield 1, None 

773 foo = 1 

774 

775 reactor = _TestReactor(events()) 

776 layer = HashableEvents() 

777 reactor.attach(layer) 

778 reactor.run() 

779 foo = 1 

780 

781def test_decoders(): 

782 """Use 2 Layers, one send a stream using random lengths 

783 The other must recompone message before sending to reactor. 

784 """ 

785 foo = 1 

786 

787if __name__ == '__main__': 

788 

789 test_decoders() 

790 test_calc() 

791 

792 # test_timeit() 

793 # test_layer() 

794 # test_await_vs_normal() 

795 test_clock() 

796 # test_hierarchy() 

797 # test_order_monitor() 

798 test_create_channel() 

799 test_crawler() 

800 # test_hashable_events() 

801 pass 

802