[18:49:24.440] src/tools/ci/cores_backtrace.sh freebsd /tmp/cores [18:49:24.861] dumping /tmp/cores/postgres.12597.core for /tmp/cirrus-ci-build/build/tmp_install/usr/local/pgsql/bin/postgres [18:49:25.006] [New LWP 100119] [18:49:25.109] Core was generated by `postgres: cascade: startup recovering 000000020000000000000004'. [18:49:25.109] Program terminated with signal SIGQUIT, Quit. [18:49:25.109] Sent by thr_kill() from pid 12597 and user 1003. [18:49:25.109] #0 0x000000082bf3e26a in _sigprocmask () from /lib/libc.so.7 [18:49:25.155] [18:49:25.155] Thread 1 (LWP 100119): [18:49:25.155] #0 0x000000082bf3e26a in _sigprocmask () from /lib/libc.so.7 [18:49:25.155] No symbol table info available. [18:49:25.155] #1 0x000000082bf79a60 in _system () from /lib/libc.so.7 [18:49:25.155] No symbol table info available. [18:49:25.155] #2 0x000000082ab240f3 in ?? () from /lib/libthr.so.3 [18:49:25.155] No symbol table info available. [18:49:25.155] #3 0x0000000000611536 in RestoreArchivedFile (path=path@entry=0x8212dc1d0 "pg_wal/00000002", '0' , "4", xlogfname=xlogfname@entry=0x8212dc5d0 "00000003.history", recovername=, expectedSize=expectedSize@entry=0, cleanupEnabled=false) at ../src/backend/access/transam/xlogarchive.c:177 [18:49:25.155] xlogpath = "pg_wal/RECOVERYHISTORY\000\000\000\020\n\000\000\000\000\000\270\301:6\b\000\000\000\300\300:6\b\000\000\000\270\300:6\b\000\000\000\270\273:6\b\000\000\000\000\340\344\000\000\000\000\000\000\360u5\b\000\000\0000\020\n\000\000\000\000\000м-!\b\000\000\000м-!\b\000\000\000\200\200\200\200\200\200\200\2009\001\000\000\000\000\000\000P\302-!\b", '\000' , "\b\000\000\0000\000\000\000\240\301-!\b\000\000\000\240\300-!\b\000\000\000\277t\344+\b\000\000\000\002\000\000\000\000\000\000\000\tt\037wc\006\000\0008\276-!\b\000\000\000d\\"... [18:49:25.155] lastRestartPointFname = '0' , '\000' , "\250C\247\000\000\000\000\000\376\377\377\377\377\377\377\377\000\000\000\000\000\000\000\000\001", '\000' , "0\000\000\000\000\000\000\000\210\300-!\b\000\000\000\264\271-!\b", '\000' , "`\271-!\b\000\000\000\250C\247\000\000\000\000\000\376\377\377\377\377\377\377\377\000\000\000\000\000\000\000\000\376\377\377\377\000\000\000\000\001\300-!\b\000\000\000\210\300-!\b\000\000\000"... [18:49:25.155] stat_buf = {st_dev = 2, st_ino = 11843528, st_nlink = 2, st_mode = 49600, st_padding0 = 8493, st_uid = 8, st_gid = 10960449, st_padding1 = 0, st_rdev = 18374403900871474943, st_atim = {tv_sec = -9187201950435737472, tv_nsec = 34916385104}, st_mtim = {tv_sec = 3, tv_nsec = 11878248}, st_ctim = {tv_sec = 35302171148, tv_nsec = 34916385088}, st_birthtim = {tv_sec = 35097121987, tv_nsec = 34916386304}, st_size = -6561262916367024127, st_blocks = -8776748686509900081, st_blksize = 78, st_flags = 0, st_gen = 2, st_spare = {0, 34916385200, 35097882096, 34916385152, 35097121789, 1710873987, 5795607548679308860, 206158430224, 34916385184, 206158430240}} [18:49:25.155] restartRedoPtr = [18:49:25.155] restartTli = [18:49:25.155] restartSegNo = [18:49:25.155] xlogRestoreCmd = 0x835728768 "cp \"/tmp/cirrus-ci-build/build/testrun/recovery/025_stuck_on_old_timeline/data/t_025_stuck_on_old_timeline_primary_data/archives/00000003.history\" \"pg_wal/RECOVERYHISTORY\"" [18:49:25.155] rc = [18:49:25.155] #4 0x00000000005f9188 in existsTimeLineHistory (probeTLI=probeTLI@entry=3) at ../src/backend/access/transam/timeline.c:235 [18:49:25.155] path = "pg_wal/00000002", '0' , "4\000\360", '\000' , "8\2607!\b\000\000\000\001\000\000\000\000\000\000\000\bT7!\b\000\000\000\000\326\340+\b\000\000\000\270\304-!\b", '\000' , "p\302-!\b\000\000\000\000\304-!\b\000\000\000\b\3406!\b\000\000\000 \000\000\000\000\000\000\000\360\303-!\b\000\000\000<\v\037wc\006\000\000\b\3406!\b\000\000\000\b\3446!\b\000\000\000\b\3506!\b\000\000\000\b\3546!\b\000\000\000\b 7!\b\000\000\000"... [18:49:25.155] histfname = "00000003.history\000\306-!\b\000\000\000\377\377\377\377\377\377\377\377P\306-!\b\000\000\000\342r\212\000\000\000\000\000\377\377\377\377\377\377\377\377\000\000\000\000\000\000\000" [18:49:25.155] fd = [18:49:25.155] #5 0x00000000005f9248 in findNewestTimeLine (startTLI=startTLI@entry=2) at ../src/backend/access/transam/timeline.c:277 [18:49:25.155] newestTLI = 2 [18:49:25.155] probeTLI = 3 [18:49:25.155] #6 0x000000000061c8bb in rescanLatestTimeLine (replayTLI=replayTLI@entry=2, replayLSN=replayLSN@entry=67209336) at ../src/backend/access/transam/xlogrecovery.c:4113 [18:49:25.155] oldtarget = 2 [18:49:25.155] currentTle = 0x0 [18:49:25.155] newtarget = [18:49:25.155] newExpectedTLEs = [18:49:25.155] found = [18:49:25.155] cell = [18:49:25.155] #7 0x0000000000618b05 in WaitForWALToBecomeAvailable (RecPtr=67209360, tliRecPtr=67209336, replayTLI=3, replayLSN=34916382640, randAccess=, fetching_ckpt=, nonblocking=) at ../src/backend/access/transam/xlogrecovery.c:3674 [18:49:25.155] startWalReceiver = false [18:49:25.155] streaming_reply_sent = true [18:49:25.155] now = [18:49:25.155] startWalReceiver = [18:49:25.155] oldSource = [18:49:25.155] wait_time = [18:49:25.155] __error = [18:49:25.155] __error = [18:49:25.155] __error = [18:49:25.155] havedata = [18:49:25.155] ptr = [18:49:25.155] tli = [18:49:25.155] __error = [18:49:25.155] latestChunkStart = [18:49:25.155] __error = [18:49:25.155] #8 XLogPageRead (xlogreader=0x83563d0b0, targetPagePtr=67207168, reqLen=2192, targetRecPtr=67209336, readBuf=0x835724f38 "\024\321\006") at ../src/backend/access/transam/xlogrecovery.c:3351 [18:49:25.155] private = 0x83563d090 [18:49:25.155] emode = 15 [18:49:25.155] targetSegNo = 4 [18:49:25.155] targetPageOff = 98304 [18:49:25.155] r = [18:49:25.155] #9 0x000000000061f857 in ReadPageInternal (state=state@entry=0x83563d0b0, pageptr=34916382640, pageptr@entry=67207168, reqLen=2192) at ../src/backend/access/transam/xlogreader.c:1060 [18:49:25.155] targetSegNo = 4 [18:49:25.155] targetPageOff = 98304 [18:49:25.155] readLen = [18:49:25.155] hdr = [18:49:25.155] err = [18:49:25.155] #10 0x000000000061e675 in XLogDecodeNextRecord (state=0x83563d0b0, nonblocking=false) at ../src/backend/access/transam/xlogreader.c:597 [18:49:25.155] randAccess = [18:49:25.155] decoded = 0x0 [18:49:25.155] RecPtr = 67209336 [18:49:25.155] assembled = false [18:49:25.155] targetPagePtr = 67207168 [18:49:25.155] readOff = [18:49:25.155] record = [18:49:25.155] gotheader = [18:49:25.155] targetRecOff = 2168 [18:49:25.155] pageHeaderSize = [18:49:25.155] len = [18:49:25.155] errormsg = [18:49:25.155] total_len = [18:49:25.155] buffer = [18:49:25.155] gotlen = [18:49:25.155] pageHeader = [18:49:25.155] contdata = [18:49:25.155] save_copy = [18:49:25.155] #11 XLogReadAhead (state=0x83563d0b0, nonblocking=false) at ../src/backend/access/transam/xlogreader.c:973 [18:49:25.155] result = [18:49:25.155] #12 0x0000000000615d9c in XLogPrefetcherNextBlock (pgsr_private=35255475936, lsn=0x835755698) at ../src/backend/access/transam/xlogprefetcher.c:492 [18:49:25.155] nonblocking = false [18:49:25.155] record = 0x0 [18:49:25.155] prefetcher = 0x83563dee0 [18:49:25.155] reader = 0x83563d0b0 [18:49:25.155] replaying_lsn = 67208928 [18:49:25.155] #13 0x0000000000615a2a in lrq_prefetch (lrq=0x835755428) at ../src/backend/access/transam/xlogprefetcher.c:252 [18:49:25.155] No locals. [18:49:25.155] #14 lrq_complete_lsn (lrq=0x835755428, lsn=) at ../src/backend/access/transam/xlogprefetcher.c:290 [18:49:25.155] No locals. [18:49:25.155] #15 XLogPrefetcherReadRecord (prefetcher=prefetcher@entry=0x83563dee0, errmsg=errmsg@entry=0x8212e0d60) at ../src/backend/access/transam/xlogprefetcher.c:1037 [18:49:25.155] replayed_up_to = [18:49:25.155] record = [18:49:25.155] #16 0x000000000061953c in ReadRecord (xlogprefetcher=0x83563dee0, emode=emode@entry=15, fetching_ckpt=false, replayTLI=replayTLI@entry=2) at ../src/backend/access/transam/xlogrecovery.c:3151 [18:49:25.155] errormsg = 0x83563d8c0 "" [18:49:25.155] xlogreader = 0x83563d0b0 [18:49:25.155] private = [18:49:25.155] record = [18:49:25.155] #17 0x0000000000619e19 in PerformWalRecovery () at ../src/backend/access/transam/xlogrecovery.c:1832 [18:49:25.155] ru0 = {tv = {tv_sec = 1710873986, tv_usec = 655109}, ru = {ru_utime = {tv_sec = 0, tv_usec = 0}, ru_stime = {tv_sec = 0, tv_usec = 1146}, ru_maxrss = 0, ru_ixrss = 0, ru_idrss = 0, ru_isrss = 0, ru_minflt = 144, ru_majflt = 0, ru_nswap = 0, ru_inblock = 0, ru_oublock = 0, ru_msgsnd = 0, ru_msgrcv = 0, ru_nsignals = 0, ru_nvcsw = 12, ru_nivcsw = 2}} [18:49:25.155] xtime = [18:49:25.155] reachedRecoveryTarget = false [18:49:25.155] replayTLI = 2 [18:49:25.155] record = [18:49:25.155] #18 0x000000000060aac9 in StartupXLOG () at ../src/backend/access/transam/xlog.c:5723 [18:49:25.155] promoted = false [18:49:25.156] didCrash = [18:49:25.156] wasShutdown = [18:49:25.156] haveTblspcMap = [18:49:25.156] haveBackupLabel = [18:49:25.156] checkPoint = {redo = 33554472, ThisTimeLineID = 1, PrevTimeLineID = , fullPageWrites = true, nextXid = {value = }, nextOid = , nextMulti = , nextMultiOffset = , oldestXid = , oldestXidDB = , oldestMulti = 1, oldestMultiDB = , time = , oldestCommitTsXid = 0, newestCommitTsXid = 0, oldestActiveXid = 738} [18:49:25.156] oldestActiveXID = [18:49:25.156] performedWalRecovery = [18:49:25.156] endOfRecoveryInfo = [18:49:25.156] EndOfLog = [18:49:25.156] EndOfLogTLI = [18:49:25.156] abortedRecPtr = [18:49:25.156] missingContrecPtr = [18:49:25.156] newTLI = [18:49:25.156] Insert = [18:49:25.156] #19 0x00000000008339fb in StartupProcessMain (startup_data=, startup_data_len=) at ../src/backend/postmaster/startup.c:258 [18:49:25.156] No locals. [18:49:25.156] #20 0x000000000082cdec in postmaster_child_launch (child_type=child_type@entry=B_STARTUP, startup_data=startup_data@entry=0x0, startup_data_len=startup_data_len@entry=0, client_sock=client_sock@entry=0x0) at ../src/backend/postmaster/launch_backend.c:267 [18:49:25.156] pid = [18:49:25.156] #21 0x000000000082f245 in StartChildProcess (type=B_AUTOVAC_WORKER, type@entry=B_STARTUP) at ../src/backend/postmaster/postmaster.c:3928 [18:49:25.156] pid = [18:49:25.156] __error = [18:49:25.156] #22 0x000000000082ee96 in PostmasterMain (argc=argc@entry=4, argv=argv@entry=0x8212e63d0) at ../src/backend/postmaster/postmaster.c:1364 [18:49:25.156] userDoption = [18:49:25.156] listen_addr_saved = false [18:49:25.156] output_config_variable = [18:49:25.156] opt = [18:49:25.156] status = [18:49:25.156] #23 0x000000000076c0b5 in main (argc=4, argv=0x8212e63d0) at ../src/backend/main/main.c:197 [18:49:25.156] do_check_root =