[10:39:01.077] src/tools/ci/cores_backtrace.sh freebsd /tmp/cores [10:39:01.468] dumping /tmp/cores/sh.12780.core for /bin/sh [10:39:01.514] [New LWP 101860] [10:39:01.514] Core was generated by `sh -c cp "/tmp/cirrus-ci-build/build/testrun/recovery/025_stuck_on_old_timeline/'. [10:39:01.514] Program terminated with signal SIGQUIT, Quit. [10:39:01.514] Sent by kill() from pid 12559 and user 1003. [10:39:01.514] #0 0x000049dd9d16aa7a in ?? () from /libexec/ld-elf.so.1 [10:39:01.522] [10:39:01.522] Thread 1 (LWP 101860): [10:39:01.522] #0 0x000049dd9d16aa7a in ?? () from /libexec/ld-elf.so.1 [10:39:01.522] No symbol table info available. [10:39:01.522] #1 0x000049dd9d166a2c in ?? () from /libexec/ld-elf.so.1 [10:39:01.522] No symbol table info available. [10:39:01.522] #2 0x0000000000000000 in ?? () [10:39:01.522] No symbol table info available. [10:39:01.529] [10:39:01.529] [10:39:01.529] [10:39:01.589] dumping /tmp/cores/postgres.16980.core for /tmp/cirrus-ci-build/build/tmp_install/usr/local/pgsql/bin/postgres [10:39:01.726] [New LWP 101868] [10:39:01.820] Core was generated by `postgres: standby: postgres postgres [local] CALL'. [10:39:01.820] Program terminated with signal SIGABRT, Aborted. [10:39:01.820] Sent by thr_kill() from pid 16980 and user 1003. [10:39:01.820] #0 0x000000082c64edca in thr_kill () from /lib/libc.so.7 [10:39:02.429] [10:39:02.429] Thread 1 (LWP 101868): [10:39:02.429] #0 0x000000082c64edca in thr_kill () from /lib/libc.so.7 [10:39:02.429] No symbol table info available. [10:39:02.429] #1 0x000000082c5c76f4 in raise () from /lib/libc.so.7 [10:39:02.429] No symbol table info available. [10:39:02.429] #2 0x000000082c678bc9 in abort () from /lib/libc.so.7 [10:39:02.429] No symbol table info available. [10:39:02.429] #3 0x0000000000a4b697 in ExceptionalCondition (conditionName=, fileName=0x30e982 "../src/backend/commands/waitlsn.c", lineNumber=lineNumber@entry=342) at ../src/backend/utils/error/assert.c:66 [10:39:02.429] No locals. [10:39:02.429] #4 0x0000000000714993 in pg_wal_replay_wait (fcinfo=) at ../src/backend/commands/waitlsn.c:342 [10:39:02.429] target_lsn = 50341200 [10:39:02.429] timeout = 0 [10:39:02.429] context = [10:39:02.429] #5 0x00000000006b7348 in ExecuteCallStmt (stmt=stmt@entry=0x836511540, params=params@entry=0x0, atomic=false, dest=0x8363a4af8) at ../src/backend/commands/functioncmds.c:2284 [10:39:02.429] fcinfodata = {fcinfo = {flinfo = 0x821949658, context = 0x8363b1610, resultinfo = 0x0, fncollation = 0, isnull = false, nargs = 2, args = 0x821949018}, fcinfo_data = "X\226\224!\b\000\000\000\020\026;6\b", '\000' , "\002\000P%\000\003\000\000\000\000\000\377\377\377\377\377\377\377\000\000\000\000\000\000\000\000\000\377\377\377\000\000\000\000\001\226\224!\b\000\000\000\260\226\224!\b\000\000\000\002\000\000\000\000\000\000\000А\224!\b\000\000\000Ɣ\251", '\000' , "`\222\224!\b\000\000\000x", '\000' , "\002\000\000\000\000\00036\360\004Q6\000\000\000\000dY3\000\000\000\000\000З\224!\b\000\000\000\260\226\224!\b\000\000\000\226Y3"...} [10:39:02.429] flinfo = {fn_addr = 0x714870 , fn_oid = 16387, fn_nargs = 2, fn_strict = true, fn_retset = false, fn_stats = 2 '\002', fn_extra = 0x0, fn_mcxt = 0x8363b1400, fn_expr = 0x836511700} [10:39:02.429] fcusage = {fs = 0x0, save_f_total_time = {ticks = 3365257}, save_total = {ticks = 4858353712}, start = {ticks = 35271019631}} [10:39:02.429] fcinfo = 0x821948ff8 [10:39:02.429] fexpr = 0x836511700 [10:39:02.429] aclresult = [10:39:02.429] callcontext = [10:39:02.429] tp = [10:39:02.429] nargs = [10:39:02.429] estate = 0x836512100 [10:39:02.429] econtext = 0x836512310 [10:39:02.429] i = 2 [10:39:02.429] lc = [10:39:02.429] retval = [10:39:02.429] #6 0x00000000008fabf6 in standard_ProcessUtility (pstmt=0x8363a45d8, queryString=0x8363a3100 "CALL pg_wal_replay_wait('0/3002550');", readOnlyTree=, context=PROCESS_UTILITY_TOPLEVEL, params=0x6, queryEnv=0x0, dest=0x8363a4af8, qc=0x821949970) at ../src/backend/tcop/utility.c:851 [10:39:02.429] isAtomicContext = true [10:39:02.429] parsetree = 0x836511540 [10:39:02.429] readonly_flags = [10:39:02.429] pstate = 0x8363b1500 [10:39:02.429] isTopLevel = [10:39:02.429] #7 0x00000000008fa7ed in ProcessUtility (pstmt=0x18dec, pstmt@entry=0x8363a45d8, queryString=0x6 , readOnlyTree=, context=744811946, params=0x0, queryEnv=0x0, dest=0x8363a4af8, qc=0x821949970) at ../src/backend/tcop/utility.c:523 [10:39:02.429] No locals. [10:39:02.429] #8 0x00000000008fa313 in PortalRunUtility (portal=0x83643a100, pstmt=0x8363a45d8, isTopLevel=, setHoldSnapshot=, dest=0x8363a4af8, qc=0x821949970) at ../src/backend/tcop/pquery.c:1158 [10:39:02.429] No locals. [10:39:02.429] #9 0x00000000008f99a4 in PortalRunMulti (portal=portal@entry=0x83643a100, isTopLevel=false, setHoldSnapshot=false, dest=dest@entry=0x8363a4af8, altdest=altdest@entry=0x8363a4af8, qc=0x821949970) at ../src/backend/tcop/pquery.c:1322 [10:39:02.429] pstmt = 0x8363a45d8 [10:39:02.429] stmtlist_item__state = {l = 0x836511930, i = 0} [10:39:02.429] active_snapshot_set = false [10:39:02.429] stmtlist_item = 0x836511948 [10:39:02.429] #10 0x00000000008f923b in PortalRun (portal=portal@entry=0x83643a100, count=count@entry=9223372036854775807, isTopLevel=false, run_once=true, dest=0x82c64edaa , dest@entry=0x8363a4af8, altdest=0x0, altdest@entry=0x8363a4af8, qc=0x821949970) at ../src/backend/tcop/pquery.c:791 [10:39:02.429] _local_sigjmp_buf = {{_sjb = {9408943, 35270441592, 34923124840, 34923125072, 34923124848, 12072280, 12040064, 35270140160, 7112098687, 35270148352, 3566469, 34359738368}}} [10:39:02.429] _save_exception_stack = 0x821949a40 [10:39:02.429] _save_context_stack = 0x0 [10:39:02.429] _do_rethrow = false [10:39:02.429] saveTopTransactionResourceOwner = 0x836483a78 [10:39:02.429] saveTopTransactionContext = 0x83642e000 [10:39:02.429] saveActivePortal = 0x0 [10:39:02.429] saveResourceOwner = 0x836483a78 [10:39:02.429] savePortalContext = 0x0 [10:39:02.429] saveMemoryContext = 0x83642e000 [10:39:02.429] result = [10:39:02.429] nprocessed = [10:39:02.429] #11 0x00000000008f811b in exec_simple_query (query_string=query_string@entry=0x8363a3100 "CALL pg_wal_replay_wait('0/3002550');") at ../src/backend/tcop/postgres.c:1274 [10:39:02.429] cmdtaglen = [10:39:02.429] snapshot_set = [10:39:02.429] per_parsetree_context = 0x0 [10:39:02.429] plantree_list = [10:39:02.429] qc = {commandTag = CMDTAG_UNKNOWN, nprocessed = 0} [10:39:02.429] parsetree = 0x8363a4378 [10:39:02.429] commandTag = CMDTAG_CALL [10:39:02.429] cmdtagname = [10:39:02.429] querytree_list = [10:39:02.429] portal = 0x83643a100 [10:39:02.429] format = 0 [10:39:02.429] receiver = 0x8363a4af8 [10:39:02.429] parsetree_item__state = {l = 0x8363a4588, i = 0} [10:39:02.429] msec_str = '\000' , "\006\000\000\000\000\000\000\000\004\000\000\000\000\000\000" [10:39:02.429] dest = DestRemote [10:39:02.429] was_logged = [10:39:02.429] oldcontext = 0x83642e000 [10:39:02.429] parsetree_list = 0x8363a4588 [10:39:02.429] use_implicit_block = false [10:39:02.429] parsetree_item = 0x8363a45a0 [10:39:02.429] save_log_statement_stats = [10:39:02.429] #12 0x00000000008f5f04 in PostgresMain (dbname=, username=) at ../src/backend/tcop/postgres.c:4676 [10:39:02.429] query_string = 0x8363a3100 "CALL pg_wal_replay_wait('0/3002550');" [10:39:02.429] input_message = {data = 0x8363a3100 "CALL pg_wal_replay_wait('0/3002550');", len = 38, maxlen = 1024, cursor = 38} [10:39:02.429] firstchar = [10:39:02.429] local_sigjmp_buf = {{_sjb = {9393104, 34923125312, 34923125272, 34923125872, 12081016, 12027488, 8192, 8388608, 34923086719, 0, 0, 34359738369}}} [10:39:02.429] send_ready_for_query = false [10:39:02.429] idle_in_transaction_timeout_enabled = false [10:39:02.429] idle_session_timeout_enabled = false [10:39:02.429] #13 0x00000000008f0c69 in BackendMain (startup_data=, startup_data_len=) at ../src/backend/tcop/backend_startup.c:101 [10:39:02.429] bsdata = [10:39:02.429] #14 0x000000000084d9f2 in postmaster_child_launch (child_type=child_type@entry=B_BACKEND, startup_data=startup_data@entry=0x82194a780 "", startup_data_len=startup_data_len@entry=4, client_sock=client_sock@entry=0x82194a6c8) at ../src/backend/postmaster/launch_backend.c:265 [10:39:02.429] pid = [10:39:02.429] #15 0x0000000000852211 in BackendStartup (client_sock=0x82194a6c8) at ../src/backend/postmaster/postmaster.c:3593 [10:39:02.429] startup_data = {canAcceptConnections = CAC_OK} [10:39:02.429] bn = 0x8363a4e30 [10:39:02.429] pid = [10:39:02.429] __error = [10:39:02.429] __error = [10:39:02.429] save_errno = [10:39:02.429] __error = [10:39:02.429] __error = [10:39:02.429] #16 ServerLoop () at ../src/backend/postmaster/postmaster.c:1674 [10:39:02.429] s = {sock = 7, raddr = {addr = {ss_len = 16 '\020', ss_family = 1 '\001', __ss_pad1 = "\000\000\000\000\000", __ss_align = 0, __ss_pad2 = '\000' , " \247\224!\b\000\000\000ؕ\326*\b\000\000\000\v\000\000\000\000\000\000\000\016\026\327*\b", '\000' , "`\247\224!\b\000\000\000\304ׄ\000\000\000\000\000p\247\224!\b", '\000' }, salen = 16}} [10:39:02.429] i = 0 [10:39:02.429] now = [10:39:02.429] events = {{pos = 1, events = 2, fd = 5, user_data = 0x0}, {pos = 743668836, events = 8, fd = 563387664, user_data = 0x821949ea0}, {pos = 566498312, events = 8, fd = 32, user_data = 0x821949e90}, {pos = -1897395564, events = 7327, fd = 566468616, user_data = 0x821c3a408}, {pos = 566470664, events = 8, fd = 566471688, user_data = 0x821c3e008}, {pos = 566486024, events = 8, fd = 566487048, user_data = 0x821c3ec08}, {pos = 566489096, events = 8, fd = 566490120, user_data = 0x821c3f808}, {pos = 566492168, events = 8, fd = 566493192, user_data = 0x821c40408}, {pos = 566495240, events = 8, fd = 566496264, user_data = 0x821c41008}, {pos = 566498312, events = 8, fd = 566517896, user_data = 0x82194a018}, {pos = 563387968, events = 8, fd = -1897370205, user_data = 0x0}, {pos = 2568774, events = 0, fd = 115249317, user_data = 0xfe76ea16}, {pos = 566521912, events = 8, fd = 1, user_data = 0x821c41408}, {pos = 743576744, events = 8, fd = 563388696, user_data = 0x821c41408}, {pos = 563387952, events = 0, fd = 743684452, user_data = 0x7ab9a6e}, {pos = 479451855, events = 8, fd = 566524472, user_data = 0x1}, {pos = 566498312, events = 8, fd = 743598296, user_data = 0x821949f58}, {pos = 563387664, events = 8, fd = 32, user_data = 0x82c537c64}, {pos = 566498312, events = 8, fd = 566524472, user_data = 0x0}, {pos = 2625, events = 0, fd = 563388224, user_data = 0x1c9f8ee80645}, {pos = 743684452, events = 8, fd = 128686702, user_data = 0x1c93dacf}, {pos = 566524472, events = 8, fd = 1, user_data = 0x821c41408}, {pos = 909717504, events = 8, fd = 1, user_data = 0x836393194}, {pos = 0, events = 0, fd = 563388176, user_data = 0x82ad70c0c}, {pos = 1, events = 0, fd = 909717504, user_data = 0x821949f40}, {pos = 718732980, events = 8, fd = 745410912, user_data = 0x82c6e1160 }, {pos = 743780544, events = 8, fd = 566498312, user_data = 0x82194a020}, {pos = -1897397204, events = 7327, fd = 566497288, user_data = 0x800000001}, {pos = -1897397586, events = 7327, fd = 12240, user_data = 0x821949f48}, {pos = 563388448, events = 8, fd = 910514152, user_data = 0x82194a590}, {pos = -1897316720, events = 7327, fd = 566498312, user_data = 0x82194037f}, {pos = 745067698, events = 8, fd = 566468616, user_data = 0x0}, {pos = 566498312, events = 8, fd = 12240, user_data = 0x82c517000}, {pos = 1964368, events = 0, fd = 745410912, user_data = 0x82c537c64}, {pos = 15616, events = 0, fd = 563389632, user_data = 0x82194a090}, {pos = 744341902, events = 8, fd = 745449528, user_data = 0x82194a054}, {pos = 910140890, events = 8, fd = 563389632, user_data = 0x82194a090}, {pos = 744433157, events = 8, fd = 563388498, user_data = 0x82194a4c0}, {pos = 563389632, events = 8, fd = 0, user_data = 0x732c527d411c0b7d}, {pos = 563389632, events = 8, fd = 910140888, user_data = 0x82194a590}, {pos = 745449528, events = 8, fd = 0, user_data = 0x82194a500}, {pos = 744433421, events = 8, fd = 0, user_data = 0x0}, {pos = 563390000, events = 8, fd = 910140888, user_data = 0x0}, {pos = 0, events = 0, fd = 0, user_data = 0x0}, {pos = 0, events = 0, fd = 0, user_data = 0x0}, {pos = -1897368567, events = 7327, fd = 563388904, user_data = 0x82c537c64}, {pos = 2503, events = 0, fd = -1897368567, user_data = 0x82194a208}, {pos = 743668836, events = 8, fd = 2758, user_data = 0x82c531120}, {pos = 563388808, events = 8, fd = 566498312, user_data = 0x82194a208}, {pos = 743641376, events = 8, fd = 563388912, user_data = 0x1c9f8ee82465}, {pos = 0, events = 0, fd = 0, user_data = 0x82c7030f0}, {pos = 743601488, events = 8, fd = 0, user_data = 0x0}, {pos = 745478576, events = 8, fd = -1897368567, user_data = 0x82194a298}, {pos = 743668836, events = 8, fd = 2924, user_data = 0x82c5313b8}, {pos = 563388952, events = 8, fd = 566498312, user_data = 0x82194a298}, {pos = 743642040, events = 8, fd = 563389056, user_data = 0x1c9f8ee82465}, {pos = 566521912, events = 8, fd = 2672274, user_data = 0xc605fbf}, {pos = 0, events = 0, fd = 0, user_data = 0x800000000}, {pos = 566498312, events = 8, fd = 743601488, user_data = 0x82194a558}, {pos = 566498312, events = 8, fd = 563389496, user_data = 0x1e}, {pos = 743605472, events = 8, fd = 563389224, user_data = 0x821c46248}, {pos = 563389672, events = 8, fd = 563389200, user_data = 0x1c9f8ee86da3}, {pos = 566521912, events = 8, fd = 2695729, user_data = 0x6d68b}, {pos = 2090263927, events = 8, fd = 566521912, user_data = 0x1}} [10:39:02.429] last_touch_time = 1711967778 [10:39:02.429] last_lockfile_recheck_time = 1711967778 [10:39:02.429] nevents = 1 [10:39:02.429] #17 0x000000000084fae6 in PostmasterMain (argc=argc@entry=4, argv=argv@entry=0x82194acc0) at ../src/backend/postmaster/postmaster.c:1372 [10:39:02.429] userDoption = [10:39:02.429] listen_addr_saved = false [10:39:02.429] output_config_variable = [10:39:02.429] opt = [10:39:02.429] status = [10:39:02.429] #18 0x000000000078ac05 in main (argc=4, argv=0x82194acc0) at ../src/backend/main/main.c:197 [10:39:02.429] do_check_root =