# Checking port 63427 # Found port 63427 Name: publisher Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/archives Connection string: port=63427 host=/tmp/PDEn8rkvNC Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_publisher.log [07:38:37.881](0.040s) # initializing database system by copying initdb template # Running: cp -RPp /tmp/cirrus-ci-build/build/tmp_install/initdb-template /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata # Running: /tmp/cirrus-ci-build/build/src/test/regress/pg_regress --config-auth /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata --create-role repl_role ### Starting node "publisher" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_publisher.log -o --cluster-name=publisher start waiting for server to start.... done server started # Postmaster PID for node "publisher" is 27459 # Checking port 63428 # Found port 63428 Name: subscriber1 Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/archives Connection string: port=63428 host=/tmp/PDEn8rkvNC Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_subscriber1.log [07:38:38.249](0.368s) # initializing database system by copying initdb template # Running: cp -RPp /tmp/cirrus-ci-build/build/tmp_install/initdb-template /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata # Running: /tmp/cirrus-ci-build/build/src/test/regress/pg_regress --config-auth /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata ### Starting node "subscriber1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_subscriber1.log -o --cluster-name=subscriber1 start waiting for server to start.... done server started # Postmaster PID for node "subscriber1" is 27536 [07:38:38.610](0.361s) ok 1 - logical slot has failover false on the publisher [07:38:38.751](0.141s) ok 2 - logical slot has failover true on the publisher [07:38:38.909](0.158s) ok 3 - logical slot has failover false on the publisher [07:38:39.056](0.147s) ok 4 - logical slot has failover true on the publisher [07:38:39.176](0.120s) ok 5 - altering failover is not allowed for enabled subscription [07:38:39.222](0.046s) ok 6 - cannot sync slots on a non-standby server # Taking pg_basebackup backup from node "publisher" # Running: pg_basebackup -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/backup/backup -h /tmp/PDEn8rkvNC -p 63427 --checkpoint fast --no-sync # Backup finished # Checking port 63429 # Found port 63429 Name: standby1 Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/archives Connection string: port=63429 host=/tmp/PDEn8rkvNC Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log # Initializing node "standby1" from backup "backup" of node "publisher" ### Enabling streaming replication for node "standby1" ### Enabling WAL restore for node "standby1" ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled (lsub2_slot,0/3000060) (sb1_slot,) ### Starting node "standby1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log -o --cluster-name=standby1 start waiting for server to start.... done server started # Postmaster PID for node "standby1" is 27799 Waiting for replication conn regress_mysub1's replay_lsn to pass 0/3000060 on publisher done Waiting for replication conn standby1's replay_lsn to pass 0/3000060 on publisher done [07:38:40.444](1.221s) ok 7 - logical slots have synced as true on standby [07:38:40.596](0.152s) ok 8 - synchronized slot has been dropped ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled Waiting for replication conn standby1's replay_lsn to pass 0/40000F8 on publisher done [07:38:41.169](0.573s) ok 9 - synchronized slot has been invalidated ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled #### Begin standard error psql::2: NOTICE: dropped replication slot "lsub1_slot" on publisher psql::3: NOTICE: created replication slot "lsub1_slot" on publisher #### End standard error Waiting for replication conn regress_mysub1's replay_lsn to pass 0/4000130 on publisher done Waiting for replication conn standby1's replay_lsn to pass 0/4000130 on publisher done [07:38:41.844](0.676s) ok 10 - logical slot is re-synced ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled [07:38:41.908](0.064s) ok 11 - logical decoding is not allowed on synced slot [07:38:41.944](0.036s) ok 12 - synced slot on standby cannot be altered [07:38:41.973](0.029s) ok 13 - synced slot on standby cannot be dropped ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled [07:38:42.049](0.075s) not ok 14 - cannot sync slots if dbname is not specified in primary_conninfo [07:38:42.049](0.000s) # Failed test 'cannot sync slots if dbname is not specified in primary_conninfo' # at /tmp/cirrus-ci-build/src/test/recovery/t/040_standby_failover_slots_sync.pl line 325. ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled # Taking pg_basebackup backup2 from node "standby1" # Running: pg_basebackup -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/backup/backup2 -h /tmp/PDEn8rkvNC -p 63429 --checkpoint fast --no-sync # Backup finished # Checking port 63430 # Found port 63430 Name: cascading_standby Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/archives Connection string: port=63430 host=/tmp/PDEn8rkvNC Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_cascading_standby.log # Initializing node "cascading_standby" from backup "backup2" of node "standby1" ### Enabling streaming replication for node "cascading_standby" ### Enabling WAL restore for node "cascading_standby" (cascading_sb_slot,) ### Starting node "cascading_standby" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_cascading_standby.log -o --cluster-name=cascading_standby start waiting for server to start.... done server started # Postmaster PID for node "cascading_standby" is 28331 [07:38:42.863](0.815s) ok 15 - cannot sync slots to a cascading standby server ### Stopping node "cascading_standby" using mode fast # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/pgdata -m fast stop waiting for server to shut down... done server stopped # No postmaster PID for node "cascading_standby" Waiting for replication conn standby1's replay_lsn to pass 0/4449810 on publisher done ### Stopping node "standby1" using mode fast # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -m fast stop waiting for server to shut down.... done server stopped # No postmaster PID for node "standby1" ### Starting node "standby1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log -o --cluster-name=standby1 start waiting for server to start.... done server started # Postmaster PID for node "standby1" is 28534 ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled Waiting for all subscriptions in "subscriber1" to synchronize data done [07:38:45.075](2.211s) ok 16 - restart_lsn and confirmed_flush_lsn of slot lsub1_slot synced to standby (sb2_slot,) # Taking pg_basebackup backup3 from node "publisher" # Running: pg_basebackup -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/backup/backup3 -h /tmp/PDEn8rkvNC -p 63427 --checkpoint fast --no-sync # Backup finished # Checking port 63431 # Found port 63431 Name: standby2 Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/archives Connection string: port=63431 host=/tmp/PDEn8rkvNC Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby2.log # Initializing node "standby2" from backup "backup3" of node "publisher" ### Enabling streaming replication for node "standby2" ### Enabling WAL restore for node "standby2" ### Starting node "standby2" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby2.log -o --cluster-name=standby2 start waiting for server to start.... done server started # Postmaster PID for node "standby2" is 28993 Waiting for replication conn standby2's replay_lsn to pass 0/6000000 on publisher done ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled # Checking port 63432 # Found port 63432 Name: subscriber2 Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/archives Connection string: port=63432 host=/tmp/PDEn8rkvNC Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_subscriber2.log [07:38:46.349](1.274s) # initializing database system by copying initdb template # Running: cp -RPp /tmp/cirrus-ci-build/build/tmp_install/initdb-template /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata # Running: /tmp/cirrus-ci-build/build/src/test/regress/pg_regress --config-auth /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata ### Starting node "subscriber2" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_subscriber2.log -o --cluster-name=subscriber2 start waiting for server to start.... done server started # Postmaster PID for node "subscriber2" is 29075 #### Begin standard error psql::3: NOTICE: created replication slot "lsub2_slot" on publisher #### End standard error Waiting for all subscriptions in "subscriber2" to synchronize data done ### Stopping node "standby1" using mode fast # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -m fast stop waiting for server to shut down.... done server stopped # No postmaster PID for node "standby1" Waiting for replication conn standby2's replay_lsn to pass 0/6000870 on publisher done [07:38:47.315](0.966s) ok 17 - standby2 gets data from primary Waiting for replication conn regress_mysub2's replay_lsn to pass 0/6000870 on publisher done [07:38:47.429](0.114s) ok 18 - subscriber2 gets data from primary [07:38:47.441](0.012s) ok 19 - subscriber1 doesn't get data from primary until standby1 acknowledges changes ### Starting node "standby1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log -o --cluster-name=standby1 start waiting for server to start.... done server started # Postmaster PID for node "standby1" is 29313 Waiting for replication conn standby1's replay_lsn to pass 0/6000870 on publisher done [07:38:47.803](0.362s) ok 20 - standby1 gets data from primary Waiting for replication conn regress_mysub1's replay_lsn to pass 0/6000870 on publisher done [07:38:47.956](0.153s) ok 21 - subscriber1 gets data from primary after standby1 acknowledges changes Waiting for replication conn regress_mysub1's replay_lsn to pass 0/6004948 on publisher done ### Stopping node "standby1" using mode fast # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -m fast stop waiting for server to shut down... done server stopped # No postmaster PID for node "standby1" ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled [07:38:49.555](1.599s) ok 22 - subscriber1 doesn't get data as the sb1_slot doesn't catch up ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled Waiting for replication conn regress_mysub1's replay_lsn to pass 0/6004F10 on publisher done [07:38:49.695](0.140s) ok 23 - subscriber1 gets data from primary after standby1 is removed from the standby_slot_names list ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled ### Starting node "standby1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log -o --cluster-name=standby1 start waiting for server to start.... done server started # Postmaster PID for node "standby1" is 29781 Waiting for replication conn standby1's replay_lsn to pass 0/6004F10 on publisher done ### Promoting node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log promote waiting for server to promote.... done server promoted [07:38:50.043](0.348s) ok 24 - synced slot retained on the new primary Waiting for replication conn regress_mysub1's replay_lsn to pass 0/6005738 on standby1 done [07:38:50.161](0.118s) ok 25 - data replicated from the new primary [07:38:50.161](0.000s) 1..25 ### Stopping node "publisher" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata -m immediate stop waiting for server to shut down... done server stopped # No postmaster PID for node "publisher" ### Stopping node "subscriber1" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata -m immediate stop waiting for server to shut down.... done server stopped # No postmaster PID for node "subscriber1" ### Stopping node "standby1" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -m immediate stop waiting for server to shut down... done server stopped # No postmaster PID for node "standby1" ### Stopping node "standby2" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/pgdata -m immediate stop waiting for server to shut down... done server stopped # No postmaster PID for node "standby2" ### Stopping node "subscriber2" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata -m immediate stop waiting for server to shut down.... done server stopped # No postmaster PID for node "subscriber2" [07:38:50.447](0.286s) # Looks like you failed 1 test of 25.