[04:49:26.218] src/tools/ci/cores_backtrace.sh freebsd /tmp/cores [04:49:26.505] dumping /tmp/cores/postgres.40959.core for /tmp/cirrus-ci-build/build/tmp_install/usr/local/pgsql/bin/postgres [04:49:26.640] [New LWP 102013] [04:49:26.725] Core was generated by `postgres: cascading_standby: startup recovering 000000020000000000000006'. [04:49:26.725] Program terminated with signal SIGQUIT, Quit. [04:49:26.725] Sent by thr_kill() from pid 40959 and user 1002. [04:49:26.725] #0 0x000000082b3d575a in _sigprocmask () from /lib/libc.so.7 [04:49:26.769] [04:49:26.769] Thread 1 (LWP 102013): [04:49:26.769] #0 0x000000082b3d575a in _sigprocmask () from /lib/libc.so.7 [04:49:26.769] No symbol table info available. [04:49:26.769] #1 0x000000082b412a2f in _system () from /lib/libc.so.7 [04:49:26.769] No symbol table info available. [04:49:26.769] #2 0x000000082ad788a3 in ?? () from /lib/libthr.so.3 [04:49:26.769] No symbol table info available. [04:49:26.769] #3 0x0000000000687347 in RestoreArchivedFile (path=path@entry=0x82170c1e0 "pg_wal/00000002", '0' , "6", xlogfname=xlogfname@entry=0x82170c5e0 "00000003.history", recovername=, expectedSize=expectedSize@entry=0, cleanupEnabled=) at ../src/backend/access/transam/xlogarchive.c:177 [04:49:26.769] xlogpath = "pg_wal/RECOVERYHISTORY\000\000\020\t;+\b\000\000\000\360\274p!\b\000\000\000`\265\326*\b\000\000\000`dG+\b", '\000' , "`\275p!\b\000\000\000$\n;+\b\000\000\000`dG+\b", '\000' , "\340\217;+\b", '\000' , "\b\000\000\000\260\301p!\b", '\000' , "\340\217;+\b", '\000' , "4%\036\002\027nTs\020\214\3148N;\000\000\001\000\000\000\000\000\000\000\200\275p!\b\000\000\000\232\262\223\000\000\000\000\000`\263H+\b\000\000\000"... [04:49:26.769] lastRestartPointFname = '0' , "\000\377\377\377\000\000\000\0008\300p!\b", '\000' , " \271p!\b\000\000\000\366e\262\000\000\000\000\000\001\000\000\000\000\000\000\0008\300p!\b\000\000\0008\300p!\b\000\000\000\377\377\377\377\000\000\000\000\001", '\000' , "\265b\262\000\000\000\000\000\240\277p!\b\000\000\000\373\377\377\377\000\000\000\000\005", '\000' , "йp!\b\000\000\000\033e\262", '\000' , "480\000\000\000\000\000\000\000\037"... [04:49:26.769] stat_buf = {st_dev = 6, st_ino = 2, st_nlink = 34920776144, st_mode = 23875, st_padding0 = 178, st_uid = 0, st_gid = 561037776, st_padding1 = 8, st_rdev = 35084631575, st_atim = {tv_sec = 34920776040, tv_nsec = 3}, st_mtim = {tv_sec = 0, tv_nsec = 0}, st_ctim = {tv_sec = 3551280, tv_nsec = 3472328296227680304}, st_birthtim = {tv_sec = 11, tv_nsec = -7842723437704380415}, st_size = 8310388257504240948, st_blocks = 35085916880, st_blksize = 5, st_flags = 0, st_gen = 0, st_spare = {34920776128, 2, 34920776080, 35085158997, 0, 1744432928, 7519666053419223005, 8310388257504240948, 206158430240, 34920776160}} [04:49:26.769] restartRedoPtr = 34920775888 [04:49:26.769] restartTli = 8 [04:49:26.769] restartSegNo = [04:49:26.769] xlogRestoreCmd = 0x3b4e38d6d3e8 "cp \"/tmp/cirrus-ci-build/build/testrun/recovery/035_standby_logical_decoding/data/t_035_standby_logical_decoding_standby_data/archives/00000003.history\" \"pg_wal/RECOVERYHISTORY\"" [04:49:26.769] rc = [04:49:26.769] #4 0x000000000066e1b8 in existsTimeLineHistory (probeTLI=probeTLI@entry=3) at ../src/backend/access/transam/timeline.c:235 [04:49:26.769] path = "pg_wal/00000002", '0' , "6\000\000\000\000\000\000\000\000\000\220P\2058N;\000\000\001\000\000\000\000\000\000\000\260\302p!\b\000\000\000\020\306p!\b\000\000\000\0200D+\b\000\000\0004%\036\002\027nTs\200\t\2408N;\000\000\000\000\000\000\000\000\000\000\220P\2058N;\000\000\001\000\000\000\000\000\000\000\260\302p!\b\000\000\000\020\306p!\b\000\000\000\2635D+\b\000\000\000\300\302p!\b", '\000' , "\n", '\000' , "\330X\2408N;\000\000\000\020\000\000\000\000\000\000h\306p!\b\000\000\000"... [04:49:26.769] histfname = "00000003.history\000\306p!\b\000\000\000\215\006\225\000\000\000\000\000\377\377\377\377\377\377\377\377\000\000\000\000\000\000\000\0000\306p!\b\000\000\000b\t=+\b\000\000" [04:49:26.769] fd = [04:49:26.769] #5 0x000000000066e278 in findNewestTimeLine (startTLI=startTLI@entry=2) at ../src/backend/access/transam/timeline.c:277 [04:49:26.769] newestTLI = 2 [04:49:26.769] probeTLI = 3 [04:49:26.769] #6 0x000000000069278b in rescanLatestTimeLine (replayTLI=replayTLI@entry=2, replayLSN=replayLSN@entry=105227432) at ../src/backend/access/transam/xlogrecovery.c:4135 [04:49:26.769] oldtarget = 2 [04:49:26.769] currentTle = 0x0 [04:49:26.769] newtarget = [04:49:26.769] newExpectedTLEs = [04:49:26.769] found = [04:49:26.769] cell = [04:49:26.769] #7 0x000000000068e983 in WaitForWALToBecomeAvailable (RecPtr=105227456, tliRecPtr=105227432, replayTLI=2, replayLSN=105227432, randAccess=, fetching_ckpt=, nonblocking=) at ../src/backend/access/transam/xlogrecovery.c:3697 [04:49:26.769] startWalReceiver = false [04:49:26.769] streaming_reply_sent = true [04:49:26.769] now = [04:49:26.769] startWalReceiver = [04:49:26.769] oldSource = [04:49:26.769] wait_time = [04:49:26.769] __error = [04:49:26.769] __error = [04:49:26.769] __error = [04:49:26.769] havedata = [04:49:26.769] ptr = [04:49:26.769] tli = [04:49:26.769] __error = [04:49:26.769] latestChunkStart = [04:49:26.769] __error = [04:49:26.769] #8 XLogPageRead (xlogreader=0x3b4e38c1d130, targetPagePtr=105226240, reqLen=1216, targetRecPtr=105227432, readBuf=0x3b4e38d0a4f8 "\030\321\006") at ../src/backend/access/transam/xlogrecovery.c:3363 [04:49:26.769] private = 0x3b4e38c1d110 [04:49:26.769] emode = 15 [04:49:26.769] targetSegNo = 6 [04:49:26.769] targetPageOff = 4562944 [04:49:26.769] io_start = [04:49:26.769] r = [04:49:26.769] #9 0x00000000006956ea in ReadPageInternal (state=state@entry=0x3b4e38c1d130, pageptr=105226240, reqLen=1216) at ../src/backend/access/transam/xlogreader.c:1061 [04:49:26.769] targetSegNo = 6 [04:49:26.769] targetPageOff = 4562944 [04:49:26.769] readLen = [04:49:26.769] hdr = [04:49:26.769] err = [04:49:26.769] #10 0x0000000000694518 in XLogDecodeNextRecord (state=0x3b4e38c1d130, nonblocking=false) at ../src/backend/access/transam/xlogreader.c:598 [04:49:26.769] errormsg = 0x821710c98 "" [04:49:26.769] randAccess = [04:49:26.769] decoded = 0x0 [04:49:26.769] RecPtr = 105227432 [04:49:26.769] assembled = false [04:49:26.769] targetPagePtr = 105226240 [04:49:26.769] readOff = [04:49:26.769] record = [04:49:26.769] gotheader = [04:49:26.769] targetRecOff = 1192 [04:49:26.769] pageHeaderSize = [04:49:26.769] len = [04:49:26.769] total_len = [04:49:26.769] buffer = [04:49:26.769] gotlen = [04:49:26.769] pageHeader = [04:49:26.769] contdata = [04:49:26.769] save_copy = [04:49:26.769] #11 XLogReadAhead (state=0x3b4e38c1d130, nonblocking=false) at ../src/backend/access/transam/xlogreader.c:974 [04:49:26.769] result = [04:49:26.769] #12 0x000000000068bc38 in XLogPrefetcherNextBlock (pgsr_private=65207146693896, lsn=0x3b4e38d6cc58) at ../src/backend/access/transam/xlogprefetcher.c:490 [04:49:26.769] nonblocking = false [04:49:26.769] record = 0x0 [04:49:26.769] prefetcher = 0x3b4e38d0c508 [04:49:26.769] reader = 0x3b4e38c1d130 [04:49:26.769] replaying_lsn = 105227384 [04:49:26.769] #13 0x000000000068b8a5 in lrq_prefetch (lrq=0x3b4e38d6cbd8) at ../src/backend/access/transam/xlogprefetcher.c:252 [04:49:26.769] No locals. [04:49:26.769] #14 lrq_complete_lsn (lrq=0x3b4e38d6cbd8, lsn=) at ../src/backend/access/transam/xlogprefetcher.c:290 [04:49:26.769] No locals. [04:49:26.769] #15 XLogPrefetcherReadRecord (prefetcher=prefetcher@entry=0x3b4e38d0c508, errmsg=errmsg@entry=0x821710d60) at ../src/backend/access/transam/xlogprefetcher.c:1035 [04:49:26.769] replayed_up_to = [04:49:26.769] record = [04:49:26.769] #16 0x000000000068f43a in ReadRecord (xlogprefetcher=0x3b4e38d0c508, emode=emode@entry=15, fetching_ckpt=false, replayTLI=replayTLI@entry=2) at ../src/backend/access/transam/xlogrecovery.c:3162 [04:49:26.769] errormsg = 0x3b4e38c1d940 "" [04:49:26.769] xlogreader = 0x3b4e38c1d130 [04:49:26.769] private = [04:49:26.769] record = [04:49:26.769] #17 0x000000000068fd29 in PerformWalRecovery () at ../src/backend/access/transam/xlogrecovery.c:1840 [04:49:26.769] ru0 = {tv = {tv_sec = 1744432926, tv_usec = 986491}, ru = {ru_utime = {tv_sec = 0, tv_usec = 0}, ru_stime = {tv_sec = 0, tv_usec = 874}, ru_maxrss = 0, ru_ixrss = 0, ru_idrss = 0, ru_isrss = 0, ru_minflt = 161, ru_majflt = 0, ru_nswap = 0, ru_inblock = 0, ru_oublock = 0, ru_msgsnd = 0, ru_msgrcv = 0, ru_nsignals = 0, ru_nvcsw = 5, ru_nivcsw = 1}} [04:49:26.769] xtime = [04:49:26.769] reachedRecoveryTarget = false [04:49:26.769] replayTLI = 2 [04:49:26.769] record = [04:49:26.769] #18 0x0000000000680433 in StartupXLOG () at ../src/backend/access/transam/xlog.c:6033 [04:49:26.769] wasShutdown = false [04:49:26.769] haveTblspcMap = false [04:49:26.769] haveBackupLabel = true [04:49:26.769] promoted = false [04:49:26.769] didCrash = [04:49:26.769] checkPoint = {redo = 100664848, ThisTimeLineID = 1, PrevTimeLineID = , fullPageWrites = true, wal_level = , nextXid = {value = }, nextOid = , nextMulti = , nextMultiOffset = , oldestXid = , oldestXidDB = , oldestMulti = , oldestMultiDB = , time = , oldestCommitTsXid = 0, newestCommitTsXid = 0, oldestActiveXid = 785} [04:49:26.769] oldestActiveXID = [04:49:26.769] performedWalRecovery = [04:49:26.769] endOfRecoveryInfo = [04:49:26.769] EndOfLog = [04:49:26.769] EndOfLogTLI = [04:49:26.769] abortedRecPtr = [04:49:26.769] missingContrecPtr = [04:49:26.769] newTLI = [04:49:26.769] Insert = [04:49:26.769] #19 0x00000000008c312b in StartupProcessMain (startup_data=, startup_data_len=) at ../src/backend/postmaster/startup.c:262 [04:49:26.769] No locals. [04:49:26.769] #20 0x00000000008bcb8b in postmaster_child_launch (child_type=child_type@entry=B_STARTUP, child_slot=77, startup_data=startup_data@entry=0x0, startup_data_len=startup_data_len@entry=0, client_sock=client_sock@entry=0x0) at ../src/backend/postmaster/launch_backend.c:290 [04:49:26.769] pid = [04:49:26.769] #21 0x00000000008bf977 in StartChildProcess (type=type@entry=B_STARTUP) at ../src/backend/postmaster/postmaster.c:3973 [04:49:26.769] pmchild = 0x0 [04:49:26.769] pid = [04:49:26.769] #22 0x00000000008bf372 in PostmasterMain (argc=argc@entry=4, argv=argv@entry=0x8217159c0) at ../src/backend/postmaster/postmaster.c:1393 [04:49:26.769] userDoption = [04:49:26.769] listen_addr_saved = false [04:49:26.769] output_config_variable = [04:49:26.769] opt = [04:49:26.769] status = [04:49:26.769] #23 0x00000000007f2510 in main (argc=4, argv=0x8217159c0) at ../src/backend/main/main.c:227 [04:49:26.769] do_check_root = [04:49:26.769] dispatch_option = DISPATCH_POSTMASTER [04:49:26.777] [04:49:26.777] [04:49:26.777] [04:49:26.832] dumping /tmp/cores/postgres.77035.core for /tmp/cirrus-ci-build/build/tmp_install/usr/local/pgsql/bin/postgres [04:49:26.960] [New LWP 101975] [04:49:27.020] Core was generated by `postgres: standby: walsender postgres [local] streaming 0/10000D0'. [04:49:27.020] Program terminated with signal SIGABRT, Aborted. [04:49:27.020] Sent by thr_kill() from pid 77035 and user 1002. [04:49:27.020] #0 0x000000082c0e11ba in thr_kill () from /lib/libc.so.7 [04:49:27.037] [04:49:27.037] Thread 1 (LWP 101975): [04:49:27.037] #0 0x000000082c0e11ba in thr_kill () from /lib/libc.so.7 [04:49:27.037] No symbol table info available. [04:49:27.037] #1 0x000000082c05a5d4 in raise () from /lib/libc.so.7 [04:49:27.037] No symbol table info available. [04:49:27.037] #2 0x000000082c10db59 in abort () from /lib/libc.so.7 [04:49:27.037] No symbol table info available. [04:49:27.037] #3 0x0000000000ad5205 in ExceptionalCondition (conditionName=0x3194b5 "pending_since == 0", fileName=, lineNumber=lineNumber@entry=734) at ../src/backend/utils/error/assert.c:66 [04:49:27.037] No locals. [04:49:27.037] #4 0x000000000098d568 in pgstat_report_stat (force=false) at ../src/backend/utils/activity/pgstat.c:734 [04:49:27.037] do_flush = [04:49:27.037] pending_since = 797748231830995 [04:49:27.037] last_flush = 797748231819967 [04:49:27.037] now = [04:49:27.037] partial_flush = [04:49:27.037] nowait = [04:49:27.037] #5 0x000000000097315b in PostgresMain (dbname=, username=) at ../src/backend/tcop/postgres.c:4627 [04:49:27.037] stats_timeout = [04:49:27.037] input_message = {data = 0x3d5828222100 "", len = 0, maxlen = 1024, cursor = 0} [04:49:27.037] firstchar = [04:49:27.037] local_sigjmp_buf = {{_sjb = {9907899, 34911224944, 34911224904, 34911225536, 12465220, 12464952, 12673912, 12667852, 895, 0, 0, 34359738369}}} [04:49:27.037] send_ready_for_query = true [04:49:27.037] idle_in_transaction_timeout_enabled = false [04:49:27.037] idle_session_timeout_enabled = false [04:49:27.037] #6 0x000000000096dfb9 in BackendMain (startup_data=, startup_data_len=) at ../src/backend/tcop/backend_startup.c:124 [04:49:27.037] bsdata = [04:49:27.037] #7 0x00000000008bcb8b in postmaster_child_launch (child_type=B_BACKEND, child_slot=1, startup_data=startup_data@entry=0x820df1128, startup_data_len=startup_data_len@entry=24, client_sock=client_sock@entry=0x820df1140) at ../src/backend/postmaster/launch_backend.c:290 [04:49:27.037] pid = [04:49:27.037] #8 0x00000000008c1508 in BackendStartup (client_sock=0x820df1140) at ../src/backend/postmaster/postmaster.c:3580 [04:49:27.037] startup_data = {canAcceptConnections = CAC_OK, socket_created = 797748231813332, fork_started = } [04:49:27.037] bn = 0x3d58283560b8 [04:49:27.037] cac = CAC_OK [04:49:27.037] pid = [04:49:27.037] __error = [04:49:27.037] save_errno = [04:49:27.037] __error = [04:49:27.037] __error = [04:49:27.037] #9 ServerLoop () at ../src/backend/postmaster/postmaster.c:1702 [04:49:27.037] s = {sock = 7, raddr = {addr = {ss_len = 16 '\020', ss_family = 1 '\001', __ss_pad1 = "\000\000\000\000\000", __ss_align = 0, __ss_pad2 = "\r", '\000' , "\240\021\337 \b\000\000\000X\306n)\b\000\000\000Y\000\000\000\000\000\000\000\276No)\b", '\000' , "\340\021\337 \b\000\000\000\304ȋ\000\000\000\000\000\360\021\337 \b", '\000' }, salen = 16}} [04:49:27.037] i = 0 [04:49:27.037] now = [04:49:27.037] events = {{pos = 1, events = 2, fd = 5, user_data = 0x0}, {pos = 0, events = 0, fd = 739754600, user_data = 0x0}, {pos = 551489472, events = 8, fd = 738743692, user_data = 0x0}, {pos = 551489776, events = 8, fd = 673789400, user_data = 0x820df0f80}, {pos = 0, events = 0, fd = 0, user_data = 0xf77ad0526e8e8833}, {pos = 739855808, events = 8, fd = 551488496, user_data = 0x1032f3aaa2cd}, {pos = 737901496, events = 8, fd = 737976476, user_data = 0x820df0c08}, {pos = 551488640, events = 8, fd = 551488544, user_data = 0x1032f3aaa2cd}, {pos = 737898352, events = 8, fd = 737976476, user_data = 0x820df0c38}, {pos = 551488688, events = 8, fd = 737947076, user_data = 0x82bfc2dc4}, {pos = 551488608, events = 8, fd = -206921011, user_data = 0x82bfbb410}, {pos = 737976476, events = 8, fd = 551488632, user_data = 0x820df0cf0}, {pos = 737950004, events = 8, fd = 737950004, user_data = 0x3d5827e0a408}, {pos = 3, events = 0, fd = 551488736, user_data = 0x1032f3aa54fb}, {pos = 551488752, events = 8, fd = 0, user_data = 0x0}, {pos = 0, events = 0, fd = 551488824, user_data = 0x820df0ef8}, {pos = 551488800, events = 8, fd = -206922670, user_data = 0x295fad}, {pos = 669053608, events = 15704, fd = 737915920, user_data = 0x3d5827e0a408}, {pos = 551488888, events = 8, fd = 551489336, user_data = 0x820df0d60}, {pos = -206922670, events = 4146, fd = 2735648, user_data = 0x6d68b}, {pos = 2090263927, events = 8, fd = 669057104, user_data = 0x800000001}, {pos = 0, events = 0, fd = 0, user_data = 0x820df1038}, {pos = 673252992, events = 3, fd = 551489416, user_data = 0x3d5827e0fae8}, {pos = 656, events = 0, fd = 551489336, user_data = 0x820df0d78}, {pos = 551488992, events = 8, fd = -206923061, user_data = 0x820df0ff8}, {pos = 0, events = 0, fd = 0, user_data = 0x0}, {pos = 669057104, events = 15704, fd = 1, user_data = 0x3d5827e0a408}, {pos = 737915920, events = 8, fd = 551489592, user_data = 0x820df0f88}, {pos = 551489008, events = 8, fd = 0, user_data = 0x3d5827e03008}, {pos = 32, events = 0, fd = 551489392, user_data = 0x1032f3aa3c77}, {pos = 669003784, events = 15704, fd = 669004808, user_data = 0x3d5827e03808}, {pos = 669006856, events = 15704, fd = 669020168, user_data = 0x3d5827e07408}, {pos = 669022216, events = 15704, fd = 669023240, user_data = 0x3d5827e08008}, {pos = 669025288, events = 15704, fd = 669026312, user_data = 0x3d5827e08c08}, {pos = 669028360, events = 15704, fd = 669029384, user_data = 0x3d5827e09808}, {pos = 669031432, events = 15704, fd = 669032456, user_data = 0x3d5827e0a408}, {pos = 669034504, events = 15704, fd = 669035528, user_data = 0x3d5827e0b008}, {pos = 669037576, events = 15704, fd = 669038600, user_data = 0x3d5827e0bc08}, {pos = 669040648, events = 15704, fd = 669041672, user_data = 0x3d5827e0c808}, {pos = 669043720, events = 15704, fd = 669048840, user_data = 0x3d5827e0e408}, {pos = 669050888, events = 15704, fd = 669051912, user_data = 0x3d5827e03008}, {pos = 2735648, events = 0, fd = 448139, user_data = 0x7c96e577}, {pos = 669057104, events = 15704, fd = 1, user_data = 0x0}, {pos = 0, events = 0, fd = 551489592, user_data = 0x820df0df0}, {pos = 32, events = 32, fd = 669003784, user_data = 0x37b}, {pos = 2113896, events = 0, fd = 297, user_data = 0x3d5827e10050}, {pos = 551489568, events = 8, fd = -206948479, user_data = 0x3d5828231000}, {pos = 2735648, events = 0, fd = 448139, user_data = 0x7c96e577}, {pos = 669057104, events = 15704, fd = 1, user_data = 0x3d5827e0a408}, {pos = 12552648, events = 0, fd = 739110672, user_data = 0x3d5827e03008}, {pos = 2839792, events = 0, fd = 695157945, user_data = 0x1}, {pos = 673259520, events = 15704, fd = 551489568, user_data = 0x8296f3642}, {pos = 739110672, events = 8, fd = 12552648, user_data = 0x82c0def10 }, {pos = 669033480, events = 15704, fd = 551489792, user_data = 0x1032f3aa3574}, {pos = 551489648, events = 8, fd = 1, user_data = 0x1032f3aa33ee}, {pos = 669003784, events = 15704, fd = 551489576, user_data = 0x820df1100}, {pos = 6816, events = 0, fd = 12, user_data = 0x1032f3ab73a0}, {pos = 0, events = 0, fd = 673317759, user_data = 0xc16370 }, {pos = 551489712, events = 8, fd = 695117152, user_data = 0x82c183460 <__thr_jtable>}, {pos = 739792720, events = 8, fd = 0, user_data = 0x1fc0}, {pos = 551489760, events = 8, fd = 695117152, user_data = 0x82c183460 <__thr_jtable>}, {pos = 739792720, events = 8, fd = 0, user_data = 0x1fc0}, {pos = 551489808, events = 8, fd = 695159549, user_data = 0x3d5828212000}, {pos = 76875, events = 0, fd = 551489904, user_data = 0x8296ec9ea}} [04:49:27.037] last_touch_time = 1744433031 [04:49:27.037] last_lockfile_recheck_time = 1744433031 [04:49:27.037] nevents = 1 [04:49:27.037] #10 0x00000000008bf3aa in PostmasterMain (argc=argc@entry=4, argv=argv@entry=0x820df1780) at ../src/backend/postmaster/postmaster.c:1400 [04:49:27.037] userDoption = [04:49:27.037] listen_addr_saved = false [04:49:27.037] output_config_variable = [04:49:27.037] opt = [04:49:27.037] status = [04:49:27.037] #11 0x00000000007f2510 in main (argc=4, argv=0x820df1780) at ../src/backend/main/main.c:227 [04:49:27.037] do_check_root = [04:49:27.037] dispatch_option = DISPATCH_POSTMASTER