[18:35:57.517] src/tools/ci/cores_backtrace.sh freebsd /tmp/cores [18:35:57.844] dumping /tmp/cores/postgres.13379.core for /tmp/cirrus-ci-build/build/tmp_install/usr/local/pgsql/bin/postgres [18:35:57.973] [New LWP 101754] [18:35:58.055] Core was generated by `postgres: cascade: startup recovering 000000020000000000000004'. [18:35:58.055] Program terminated with signal SIGQUIT, Quit. [18:35:58.055] Sent by thr_kill() from pid 13379 and user 1003. [18:35:58.055] #0 0x000000082befb8fa in _sigprocmask () from /lib/libc.so.7 [18:35:58.102] [18:35:58.102] Thread 1 (LWP 101754): [18:35:58.102] #0 0x000000082befb8fa in _sigprocmask () from /lib/libc.so.7 [18:35:58.102] No symbol table info available. [18:35:58.102] #1 0x000000082bf34f91 in _system () from /lib/libc.so.7 [18:35:58.102] No symbol table info available. [18:35:58.102] #2 0x000000082b5089d3 in ?? () from /lib/libthr.so.3 [18:35:58.102] No symbol table info available. [18:35:58.102] #3 0x000000000066a0e5 in RestoreArchivedFile (path=path@entry=0x8217431d0 "pg_wal/00000002", '0' , "4", xlogfname=xlogfname@entry=0x8217435d0 "00000003.history", recovername=, expectedSize=expectedSize@entry=0, cleanupEnabled=) at ../src/backend/access/transam/xlogarchive.c:177 [18:35:58.102] xlogpath = "pg_wal/RECOVERYHISTORY", '\000' , "\200\t@6\b\000\000\000\2106t!\b\000\000\0009\000\000\000\000\000\000\000\000\000\344\000\000\000\000\000\000\020\n\000\000\000\000\000\200\315@6\b\000\000\000P\255[8\b\000\000\000p\254[8\b\000\000\000h\254[8\b\000\000\000h\247[8\b\000\000\000\000\340\344\000\000\000\000\000\000PM7\b\000\000\000\000\020\n\000\000\000\000\000 -t!\b\000\000\000 -t!\b", '\000' , "\350.t!\b\000\000\000h-t!\b\000\000\000\b\000\000\0000\000\000\000"... [18:35:58.102] lastRestartPointFname = '0' , "\000\000\000\000\000\000\000\000\030\000\000\0000\000\000\000\260,t!\b\000\000\000`*t!\b", '\000' , "\240\326\372+\b", '\000' , "\265\034\257\000\000\000\000\000\300)t!\b\000\000\0000\000\000\000\000\000\000\000\2100t!\b", '\000' , "p)t!\b\000\000\000\003 \257", '\000' , "\265\034\257\000\000\000\000\000\2100t!\b\000\000\000\376\377\377\377\000\000\000\000\002", '\000' ... [18:35:58.102] stat_buf = {st_dev = 98304, st_ino = 2, st_nlink = 34921001408, st_mode = 5955, st_padding0 = 175, st_uid = 0, st_gid = 561263040, st_padding1 = 8, st_rdev = 35096323559, st_atim = {tv_sec = 12567, tv_nsec = 3}, st_mtim = {tv_sec = 35342413116, tv_nsec = 27720}, st_ctim = {tv_sec = -72340172838076673, tv_nsec = -9187201950435737472}, st_birthtim = {tv_sec = 7864918215260855397, tv_nsec = 34921001392}, st_size = 6, st_blocks = 0, st_blksize = 78, st_flags = 0, st_gen = 35097597600, st_spare = {34921001328, 35096849071, 1737829795, 9730204100960086643, 7864918215260855397, 2, 0, 34921001392, 206158430240, 34921001424}} [18:35:58.102] restartRedoPtr = 34921001152 [18:35:58.102] restartTli = 8 [18:35:58.102] restartSegNo = [18:35:58.102] xlogRestoreCmd = 0x83749d9b8 "cp \"/tmp/cirrus-ci-build/build/testrun/recovery/025_stuck_on_old_timeline/data/t_025_stuck_on_old_timeline_primary_data/archives/00000003.history\" \"pg_wal/RECOVERYHISTORY\"" [18:35:58.102] rc = [18:35:58.102] #4 0x00000000006511c8 in existsTimeLineHistory (probeTLI=probeTLI@entry=3) at ../src/backend/access/transam/timeline.c:235 [18:35:58.102] path = "pg_wal/00000002", '0' , "4", '\000' , "8\200\324!\b\000\000\000\001\000\000\000\000\000\000\000\b$\324!\b\000\000\0008\342\334+\b\000\000\000\2504t!\b\000\000\000\3603t!\b\000\000\000`2t!\b", '\000' , "\b\260\323!\b\000\000\000 \000\000\000\000\000\000\000\3403t!\b\000\000\000\347\272M\3368\004\000\000\b\260\323!\b\000\000\000\b\264\323!\b\000\000\000\b\270\323!\b\000\000\000\b\274\323!\b\000\000\000\b\360\323!\b\000\000\000\b\364\323!\b\000\000\000\b\370\323!\b\000\000\000"... [18:35:58.102] histfname = "00000003.history\0006t!\b\000\000\000OR\221", '\000' , "\377\377\377\377\377\377\377\377e\\\037Xr\315%m\000\000\000\000\000\000\000" [18:35:58.102] fd = [18:35:58.102] #5 0x0000000000651288 in findNewestTimeLine (startTLI=startTLI@entry=2) at ../src/backend/access/transam/timeline.c:277 [18:35:58.102] newestTLI = 2 [18:35:58.102] probeTLI = 3 [18:35:58.102] #6 0x000000000067550b in rescanLatestTimeLine (replayTLI=replayTLI@entry=2, replayLSN=replayLSN@entry=67210792) at ../src/backend/access/transam/xlogrecovery.c:4115 [18:35:58.102] oldtarget = 2 [18:35:58.102] currentTle = 0x0 [18:35:58.102] newtarget = [18:35:58.102] newExpectedTLEs = [18:35:58.102] found = [18:35:58.102] cell = [18:35:58.102] #7 0x0000000000671753 in WaitForWALToBecomeAvailable (RecPtr=67210816, tliRecPtr=67210792, replayTLI=2, replayLSN=67210792, randAccess=, fetching_ckpt=, nonblocking=) at ../src/backend/access/transam/xlogrecovery.c:3677 [18:35:58.102] startWalReceiver = false [18:35:58.102] streaming_reply_sent = true [18:35:58.102] now = [18:35:58.102] startWalReceiver = [18:35:58.102] oldSource = [18:35:58.102] wait_time = [18:35:58.102] __error = [18:35:58.102] __error = [18:35:58.102] __error = [18:35:58.102] havedata = [18:35:58.102] ptr = [18:35:58.102] tli = [18:35:58.102] __error = [18:35:58.102] latestChunkStart = [18:35:58.102] __error = [18:35:58.102] #8 XLogPageRead (xlogreader=0x8373b0130, targetPagePtr=67207168, reqLen=3648, targetRecPtr=67210792, readBuf=0x83749a078 "\030\321\006") at ../src/backend/access/transam/xlogrecovery.c:3353 [18:35:58.102] private = 0x8373b0110 [18:35:58.102] emode = 15 [18:35:58.102] targetSegNo = 4 [18:35:58.102] targetPageOff = 98304 [18:35:58.102] r = [18:35:58.102] #9 0x0000000000678452 in ReadPageInternal (state=state@entry=0x8373b0130, pageptr=67207168, reqLen=3648) at ../src/backend/access/transam/xlogreader.c:1060 [18:35:58.102] targetSegNo = 4 [18:35:58.102] targetPageOff = 98304 [18:35:58.102] readLen = [18:35:58.102] hdr = [18:35:58.102] err = [18:35:58.102] #10 0x00000000006772c2 in XLogDecodeNextRecord (state=0x8373b0130, nonblocking=false) at ../src/backend/access/transam/xlogreader.c:597 [18:35:58.102] errormsg = 0x821747c98 "" [18:35:58.102] randAccess = [18:35:58.102] decoded = 0x0 [18:35:58.102] RecPtr = 67210792 [18:35:58.102] assembled = false [18:35:58.102] targetPagePtr = 34920998848 [18:35:58.102] readOff = [18:35:58.102] record = [18:35:58.102] gotheader = [18:35:58.102] targetRecOff = 3624 [18:35:58.102] pageHeaderSize = [18:35:58.102] len = [18:35:58.102] total_len = [18:35:58.102] buffer = [18:35:58.102] gotlen = [18:35:58.102] pageHeader = [18:35:58.102] contdata = [18:35:58.102] save_copy = [18:35:58.102] #11 XLogReadAhead (state=0x8373b0130, nonblocking=false) at ../src/backend/access/transam/xlogreader.c:973 [18:35:58.102] result = [18:35:58.102] #12 0x000000000066ea08 in XLogPrefetcherNextBlock (pgsr_private=35287318664, lsn=0x8374cabc8) at ../src/backend/access/transam/xlogprefetcher.c:490 [18:35:58.102] nonblocking = false [18:35:58.102] record = 0x0 [18:35:58.102] prefetcher = 0x83749c088 [18:35:58.102] reader = 0x8373b0130 [18:35:58.102] replaying_lsn = 67210384 [18:35:58.102] #13 0x000000000066e672 in lrq_prefetch (lrq=0x8374ca928) at ../src/backend/access/transam/xlogprefetcher.c:252 [18:35:58.102] No locals. [18:35:58.102] #14 lrq_complete_lsn (lrq=0x8374ca928, lsn=) at ../src/backend/access/transam/xlogprefetcher.c:290 [18:35:58.102] No locals. [18:35:58.102] #15 XLogPrefetcherReadRecord (prefetcher=prefetcher@entry=0x83749c088, errmsg=errmsg@entry=0x821747d60) at ../src/backend/access/transam/xlogprefetcher.c:1035 [18:35:58.102] replayed_up_to = [18:35:58.102] record = [18:35:58.102] #16 0x00000000006721aa in ReadRecord (xlogprefetcher=0x83749c088, emode=emode@entry=15, fetching_ckpt=false, replayTLI=replayTLI@entry=2) at ../src/backend/access/transam/xlogrecovery.c:3153 [18:35:58.102] errormsg = 0x8373b0940 "" [18:35:58.102] xlogreader = 0x8373b0130 [18:35:58.102] private = [18:35:58.102] record = [18:35:58.102] #17 0x0000000000672a89 in PerformWalRecovery () at ../src/backend/access/transam/xlogrecovery.c:1832 [18:35:58.102] ru0 = {tv = {tv_sec = 1737829794, tv_usec = 859870}, ru = {ru_utime = {tv_sec = 0, tv_usec = 0}, ru_stime = {tv_sec = 0, tv_usec = 953}, ru_maxrss = 0, ru_ixrss = 0, ru_idrss = 0, ru_isrss = 0, ru_minflt = 149, ru_majflt = 0, ru_nswap = 0, ru_inblock = 0, ru_oublock = 0, ru_msgsnd = 0, ru_msgrcv = 0, ru_nsignals = 0, ru_nvcsw = 12, ru_nivcsw = 2}} [18:35:58.102] xtime = [18:35:58.102] reachedRecoveryTarget = false [18:35:58.102] replayTLI = 2 [18:35:58.102] record = [18:35:58.102] #18 0x0000000000663413 in StartupXLOG () at ../src/backend/access/transam/xlog.c:5848 [18:35:58.102] wasShutdown = false [18:35:58.102] haveTblspcMap = false [18:35:58.102] haveBackupLabel = true [18:35:58.102] promoted = false [18:35:58.102] didCrash = [18:35:58.102] checkPoint = {redo = 33554472, ThisTimeLineID = 1, PrevTimeLineID = , fullPageWrites = true, wal_level = , nextXid = {value = }, nextOid = , nextMulti = , nextMultiOffset = , oldestXid = , oldestXidDB = 1, oldestMulti = , oldestMultiDB = , time = , oldestCommitTsXid = 0, newestCommitTsXid = 0, oldestActiveXid = 743} [18:35:58.102] oldestActiveXID = [18:35:58.102] performedWalRecovery = [18:35:58.102] endOfRecoveryInfo = [18:35:58.102] EndOfLog = [18:35:58.102] EndOfLogTLI = [18:35:58.102] abortedRecPtr = [18:35:58.102] missingContrecPtr = [18:35:58.102] newTLI = [18:35:58.102] Insert = [18:35:58.102] #19 0x000000000089baab in StartupProcessMain (startup_data=, startup_data_len=) at ../src/backend/postmaster/startup.c:258 [18:35:58.102] No locals. [18:35:58.102] #20 0x0000000000895e45 in postmaster_child_launch (child_type=child_type@entry=B_STARTUP, child_slot=57, startup_data=startup_data@entry=0x0, startup_data_len=startup_data_len@entry=0, client_sock=client_sock@entry=0x0) at ../src/backend/postmaster/launch_backend.c:274 [18:35:58.102] pid = [18:35:58.102] #21 0x0000000000898a47 in StartChildProcess (type=type@entry=B_STARTUP) at ../src/backend/postmaster/postmaster.c:3905 [18:35:58.102] pmchild = 0x0 [18:35:58.102] pid = [18:35:58.102] #22 0x00000000008985e2 in PostmasterMain (argc=argc@entry=4, argv=argv@entry=0x82174d320) at ../src/backend/postmaster/postmaster.c:1378 [18:35:58.102] userDoption = [18:35:58.102] listen_addr_saved = false [18:35:58.102] output_config_variable = [18:35:58.102] opt = [18:35:58.102] status = [18:35:58.102] #23 0x00000000007cee30 in main (argc=4, argv=0x82174d320) at ../src/backend/main/main.c:230 [18:35:58.102] do_check_root = [18:35:58.102] dispatch_option = DISPATCH_POSTMASTER [18:35:58.112] [18:35:58.112] [18:35:58.112] [18:35:58.176] dumping /tmp/cores/postgres.21596.core for /tmp/cirrus-ci-build/build/tmp_install//usr/local/pgsql/bin/postgres [18:35:58.299] [New LWP 101842] [18:35:58.362] Core was generated by `postgres: postgres isolation_regression_injection_points [local] COMMIT'. [18:35:58.362] Program terminated with signal SIGABRT, Aborted. [18:35:58.362] Sent by thr_kill() from pid 21596 and user 1003. [18:35:58.362] #0 0x000000082af6e41a in thr_kill () from /lib/libc.so.7 [18:35:58.383] [18:35:58.383] Thread 1 (LWP 101842): [18:35:58.383] #0 0x000000082af6e41a in thr_kill () from /lib/libc.so.7 [18:35:58.383] No symbol table info available. [18:35:58.383] #1 0x000000082aee7e64 in raise () from /lib/libc.so.7 [18:35:58.383] No symbol table info available. [18:35:58.383] #2 0x000000082af986f9 in abort () from /lib/libc.so.7 [18:35:58.383] No symbol table info available. [18:35:58.383] #3 0x0000000000aa2975 in ExceptionalCondition (conditionName=0x361cdc "TransactionIdIsValid(proc->xid)", fileName=, lineNumber=lineNumber@entry=677) at ../src/backend/utils/error/assert.c:66 [18:35:58.383] No locals. [18:35:58.383] #4 0x0000000000916d5d in ProcArrayEndTransaction (proc=0x844429378, latestXid=latestXid@entry=751) at ../src/backend/storage/ipc/procarray.c:677 [18:35:58.383] No locals. [18:35:58.383] #5 0x000000000065a38f in AbortTransaction () at ../src/backend/access/transam/xact.c:2946 [18:35:58.383] s = [18:35:58.383] latestXid = 751 [18:35:58.383] is_parallel_worker = [18:35:58.383] #6 0x0000000000657f9a in AbortCurrentTransactionInternal () at ../src/backend/access/transam/xact.c:3598 [18:35:58.383] s = 0xbe7408 [18:35:58.383] #7 AbortCurrentTransaction () at ../src/backend/access/transam/xact.c:3449 [18:35:58.383] No locals. [18:35:58.383] #8 0x00000000009431de in PostgresMain (dbname=, username=) at ../src/backend/tcop/postgres.c:4406 [18:35:58.383] local_sigjmp_buf = {{_sjb = {9711990, 34924799504, 34924799496, 34924800048, 12570616, 1, 8192, 8388608, 895, 0, 0, 34359738369}}} [18:35:58.383] send_ready_for_query = false [18:35:58.383] idle_in_transaction_timeout_enabled = false [18:35:58.383] idle_session_timeout_enabled = false [18:35:58.383] #9 0x000000000093e669 in BackendMain (startup_data=, startup_data_len=) at ../src/backend/tcop/backend_startup.c:107 [18:35:58.383] bsdata = [18:35:58.383] #10 0x0000000000895e45 in postmaster_child_launch (child_type=B_BACKEND, child_slot=1, startup_data=startup_data@entry=0x821ae3340 "", startup_data_len=startup_data_len@entry=4, client_sock=client_sock@entry=0x821ae3290) at ../src/backend/postmaster/launch_backend.c:274 [18:35:58.383] pid = [18:35:58.383] #11 0x000000000089a1eb in BackendStartup (client_sock=0x821ae3290) at ../src/backend/postmaster/postmaster.c:3519 [18:35:58.383] startup_data = {canAcceptConnections = CAC_OK} [18:35:58.383] bn = 0x8369c9038 [18:35:58.383] cac = CAC_OK [18:35:58.383] pid = [18:35:58.383] __error = [18:35:58.383] save_errno = [18:35:58.383] __error = [18:35:58.383] __error = [18:35:58.383] #12 ServerLoop () at ../src/backend/postmaster/postmaster.c:1688 [18:35:58.383] s = {sock = 8, raddr = {addr = {ss_len = 16 '\020', ss_family = 1 '\001', __ss_pad1 = "\000\000\000\000\000", __ss_align = 0, __ss_pad2 = "\f", '\000' , "\3602\256!\b\000\000\000h\3652*\b\000\000\000\372\000\000\000\000\000\000\000^\1773*\b", '\000' , "03\256!\b\000\000\000\324[\211\000\000\000\000\000@3\256!\b", '\000' }, salen = 16}} [18:35:58.383] i = 0 [18:35:58.383] now = [18:35:58.383] events = {{pos = 1, events = 2, fd = 6, user_data = 0x0}, {pos = 0, events = 0, fd = 0, user_data = 0x0}, {pos = 915940824, events = 8, fd = 565063888, user_data = 0x821ae2e30}, {pos = 8192, events = 0, fd = 8192, user_data = 0x8369f5000}, {pos = 565063224, events = 8, fd = 0, user_data = 0x83406f090}, {pos = 1, events = 0, fd = 565063024, user_data = 0x82afdd9c9}, {pos = 719594376, events = 8, fd = 719687032, user_data = 0x821ae2d88}, {pos = 565063168, events = 8, fd = 565063216, user_data = 0x821ae2e28}, {pos = 872870032, events = 8, fd = 908069248, user_data = 0x20}, {pos = 4096, events = 0, fd = 565063664, user_data = 0x82afd8a7c}, {pos = 565063208, events = 8, fd = -539925750, user_data = 0x82ae3a858}, {pos = 719687032, events = 8, fd = 565063160, user_data = 0x821ae2e70}, {pos = 565063160, events = 8, fd = 719650060, user_data = 0x821b76408}, {pos = 719650060, events = 8, fd = 565063264, user_data = 0x2cb8dfd1143a}, {pos = 872870080, events = 8, fd = 4096, user_data = 0x1001}, {pos = 0, events = 0, fd = 0, user_data = 0x800000000}, {pos = 908079672, events = 8, fd = 0, user_data = 0x0}, {pos = 0, events = 0, fd = 0, user_data = 0x821b7b288}, {pos = 719562840, events = 8, fd = 565666824, user_data = 0x821ae2ef8}, {pos = 565063864, events = 8, fd = 565063392, user_data = 0x2cb8dfd15ca2}, {pos = 2722030, events = 0, fd = 448139, user_data = 0x7c96e577}, {pos = 565690424, events = 8, fd = 1, user_data = 0x0}, {pos = 0, events = 0, fd = 565064120, user_data = 0x21b76408}, {pos = 565063936, events = 8, fd = 565063536, user_data = 0x1}, {pos = 565063864, events = 8, fd = 32, user_data = 0x821ae2f60}, {pos = -539927949, events = 11448, fd = 565063808, user_data = 0x0}, {pos = 0, events = 0, fd = 0, user_data = 0x821b7c038}, {pos = 1, events = 8, fd = 565666824, user_data = 0x82ae3a858}, {pos = 565064120, events = 8, fd = 565063936, user_data = 0x821ae2f70}, {pos = 0, events = 0, fd = 565637128, user_data = 0x20}, {pos = 565063920, events = 8, fd = -539952409, user_data = 0x821b6f008}, {pos = 565638152, events = 8, fd = 565639176, user_data = 0x821b6fc08}, {pos = 565653512, events = 8, fd = 565654536, user_data = 0x821b73808}, {pos = 565656584, events = 8, fd = 565657608, user_data = 0x821b74408}, {pos = 565659656, events = 8, fd = 565660680, user_data = 0x821b75008}, {pos = 565662728, events = 8, fd = 565663752, user_data = 0x821b75c08}, {pos = 565665800, events = 8, fd = 565666824, user_data = 0x821b76808}, {pos = 565668872, events = 8, fd = 565669896, user_data = 0x821b77408}, {pos = 565671944, events = 8, fd = 565672968, user_data = 0x821b78008}, {pos = 565675016, events = 8, fd = 565676040, user_data = 0x821b78c08}, {pos = 565682184, events = 8, fd = 565683208, user_data = 0x821b7a808}, {pos = 565685256, events = 8, fd = 565637128, user_data = 0x2988ee}, {pos = 448139, events = 0, fd = 2090263927, user_data = 0x821b7c038}, {pos = 1, events = 0, fd = 0, user_data = 0x0}, {pos = 565064120, events = 8, fd = 565063536, user_data = 0x2000000020}, {pos = 0, events = 0, fd = 565690424, user_data = 0x1}, {pos = 565637128, events = 8, fd = 298, user_data = 0x821ae31a0}, {pos = -539953681, events = 11448, fd = 2722030, user_data = 0x6d68b}, {pos = 2090263927, events = 0, fd = 565690424, user_data = 0x1}, {pos = 565666824, events = 8, fd = 915505152, user_data = 0x1}, {pos = 915505556, events = 8, fd = 0, user_data = 0x821ae3170}, {pos = 708015468, events = 8, fd = 1, user_data = 0x836918000}, {pos = 565064096, events = 8, fd = 708011778, user_data = 0x82af6c170 }, {pos = 12478704, events = 0, fd = 720814448, user_data = 0x821b76408}, {pos = 565064320, events = 8, fd = -539954210, user_data = 0x836936800}, {pos = 1, events = 8, fd = -539954594, user_data = 0x821b6f008}, {pos = 565064104, events = 8, fd = 565064320, user_data = 0x1ab8}, {pos = 11, events = 0, fd = -539874928, user_data = 0x0}, {pos = 565052287, events = 8, fd = 8134999, user_data = 0x821ae3230}, {pos = 707974160, events = 8, fd = 721477680, user_data = 0x82b010310}, {pos = 0, events = 0, fd = 8128, user_data = 0x821ae3260}, {pos = 708017053, events = 8, fd = 915505152, user_data = 0x5438}, {pos = 565064384, events = 8, fd = 707983610, user_data = 0x0}, {pos = 0, events = 0, fd = 1206338349, user_data = 0x0}} [18:35:58.383] last_touch_time = 1737829864 [18:35:58.383] last_lockfile_recheck_time = 1737829864 [18:35:58.383] nevents = 1 [18:35:58.383] #13 0x0000000000898624 in PostmasterMain (argc=argc@entry=8, argv=argv@entry=0x821ae38b0) at ../src/backend/postmaster/postmaster.c:1386 [18:35:58.383] userDoption = [18:35:58.383] listen_addr_saved = false [18:35:58.383] output_config_variable = [18:35:58.383] opt = [18:35:58.383] status = [18:35:58.383] #14 0x00000000007cee30 in main (argc=8, argv=0x821ae38b0) at ../src/backend/main/main.c:230 [18:35:58.383] do_check_root = [18:35:58.383] dispatch_option = DISPATCH_POSTMASTER