From 823a575d9a70ed7f2cb08be462c09a399a4a2842 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 19 Feb 2026 21:27:17 +1030 Subject: [PATCH] pytest: fix flake involving partial lines. Not sure how this happens, but it does, and may explain other races: ``` line = l1.daemon.wait_for_log(f"plugin-all_notifications.py: notification pay_failure: ") dict_str = line.split("notification pay_failure: ", 1)[1] > data = ast.literal_eval(dict_str) tests/test_plugin.py:4869: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /opt/hostedtoolcache/Python/3.10.19/x64/lib/python3.10/ast.py:64: in literal_eval node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval') _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ source = "{'origin': 'pay', 'payload': {'payment_hash': 'a9caff840abedac582c3e22b46f78d205d8" filename = '', mode = 'eval' def parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). Pass type_comments=True to get back type comments where the syntax allows. """ flags = PyCF_ONLY_AST if type_comments: flags |= PyCF_TYPE_COMMENTS if isinstance(feature_version, tuple): major, minor = feature_version # Should be a 2-tuple. assert major == 3 feature_version = minor elif feature_version is None: feature_version = -1 # Else it should be an int giving the minor version for 3.x. > return compile(source, filename, mode, flags, _feature_version=feature_version) E File "", line 1 E {'origin': 'pay', 'payload': {'payment_hash': 'a9caff840abedac582c3e22b46f78d205d8 E ^ E SyntaxError: unterminated string literal (detected at line 1) /opt/hostedtoolcache/Python/3.10.19/x64/lib/python3.10/ast.py:50: SyntaxError ... lightningd-1 2026-02-19T03:32:31.425Z INFO plugin-all_notifications.py: notification pay_failure: {'origin': 'pay', 'payload': {'payment_hash': 'a9caff840abedac582c3e22b46f78d205d87ff1212de05a64b12a6a53459bf29', 'bolt11': 'lnbcrt100n1p5edpz0sp5l5nj5p28kzm47d0tug2c2hz9q9gknd5sve6nwkf59fnpfvzecvcqpp54890lpq2hmdvtqkrug45daudypwc0lcjzt0qtfjtz2n22dzehu5sdq8v3jhxccxqyjw5qcqp9rzjqvuytqpdyk6wqaxvl47d3vee5swuwklej79qxjqqg394r4ptqaue5qqqvuqqqqgqqqqqqqqpqqqqqzsqqc9qxpqysgqvqzc5wd097xa6shdfkvr3yarvj36gu00ylfwdlfql9wkq3kueqgkk3pa3zdzd8a0de5vny0whzwjzrvhcuf86q6m9yakwjwjs3hzl9gpcf5lzn', 'error': {'message': 'failed: WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS (reply from remote)'}}, 'pay_failure': {'payment_hash': 'a9caff840abedac582c3e22b46f78d205d87ff1212de05a64b12a6a53459bf29', 'bolt11': 'lnbcrt100n1p5edpz0sp5l5nj5p28kzm47d0tug2c2hz9q9gknd5sve6nwkf59fnpfvzecvcqpp54890lpq2hmdvtqkrug45daudypwc0lcjzt0qtfjtz2n22dzehu5sdq8v3jhxccxqyjw5qcqp9rzjqvuytqpdyk6wqaxvl47d3vee5swuwklej79qxjqqg394r4ptqaue5qqqvuqqqqgqqqqqqqqpqqqqqzsqqc9qxpqysgqvqzc5wd097xa6shdfkvr3yarvj36gu00ylfwdlfql9wkq3kueqgkk3pa3zdzd8a0de5vny0whzwjzrvhcuf86q6m9yakwjwjs3hzl9gpcf5lzn', 'error': {'message': 'failed: WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS (reply from remote)'}}} ``` Signed-off-by: Rusty Russell --- contrib/pyln-testing/pyln/testing/utils.py | 32 ++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index ce2a1007c..dd60020cc 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -292,15 +292,43 @@ class TailableProc(object): except Exception: pass + def readlines_wait_for_end(self, f, timeout=TIMEOUT): + """Read all complete lines from file object `f`. + + If the last line is incomplete (no trailing newline), wait briefly + for it to complete before returning. + + Returns list of lines including trailing newline. + """ + lines = [] + cur = '' + start = time.time() + + while True: + line = f.readline() + + if not line: + if cur != '': + if time.time() - start > timeout: + raise TimeoutError(f"Incomplete line never finished: {cur}") + time.sleep(0.01) + continue + return lines + + cur += line + if cur.endswith('\n'): + lines.append(cur) + cur = '' + def logs_catchup(self): """Save the latest stdout / stderr contents; return true if we got anything. """ - new_stdout = self.stdout_read.readlines() + new_stdout = self.readlines_wait_for_end(self.stdout_read) if self.verbose: for line in new_stdout: sys.stdout.write("{}: {}".format(self.prefix, line)) self.logs += [l.rstrip() for l in new_stdout] - new_stderr = self.stderr_read.readlines() + new_stderr = self.readlines_wait_for_end(self.stderr_read) if self.verbose: for line in new_stderr: sys.stderr.write("{}-stderr: {}".format(self.prefix, line))