# Checking port 50630 # Found port 50630 Name: publisher Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/archives Connection string: port=50630 host=/tmp/t9VPlUiE88 Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_publisher.log [07:37:50.582](0.025s) # initializing database system by copying initdb template # Running: cp -RPp /tmp/cirrus-ci-build/build/tmp_install/initdb-template /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata # Running: /tmp/cirrus-ci-build/build/src/test/regress/pg_regress --config-auth /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata --create-role repl_role ### Starting node "publisher" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_publisher.log -o --cluster-name=publisher start waiting for server to start.... done server started # Postmaster PID for node "publisher" is 14623 # Checking port 50631 # Found port 50631 Name: subscriber1 Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/archives Connection string: port=50631 host=/tmp/t9VPlUiE88 Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_subscriber1.log [07:37:50.802](0.219s) # initializing database system by copying initdb template # Running: cp -RPp /tmp/cirrus-ci-build/build/tmp_install/initdb-template /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata # Running: /tmp/cirrus-ci-build/build/src/test/regress/pg_regress --config-auth /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata ### Starting node "subscriber1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_subscriber1.log -o --cluster-name=subscriber1 start waiting for server to start.... done server started # Postmaster PID for node "subscriber1" is 14684 [07:37:51.020](0.218s) ok 1 - logical slot has failover false on the publisher [07:37:51.066](0.046s) ok 2 - logical slot has failover true on the publisher [07:37:51.103](0.037s) ok 3 - logical slot has failover false on the publisher [07:37:51.141](0.038s) ok 4 - logical slot has failover true on the publisher [07:37:51.159](0.018s) ok 5 - altering failover is not allowed for enabled subscription [07:37:51.168](0.009s) ok 6 - cannot sync slots on a non-standby server # Taking pg_basebackup backup from node "publisher" # Running: pg_basebackup -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/backup/backup -h /tmp/t9VPlUiE88 -p 50630 --checkpoint fast --no-sync # Backup finished # Checking port 50632 # Found port 50632 Name: standby1 Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/archives Connection string: port=50632 host=/tmp/t9VPlUiE88 Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log # Initializing node "standby1" from backup "backup" of node "publisher" ### Enabling streaming replication for node "standby1" ### Enabling WAL restore for node "standby1" ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled (lsub2_slot,0/3000060) (sb1_slot,) ### Starting node "standby1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log -o --cluster-name=standby1 start waiting for server to start.... done server started # Postmaster PID for node "standby1" is 14841 Waiting for replication conn regress_mysub1's replay_lsn to pass 0/3000060 on publisher done Waiting for replication conn standby1's replay_lsn to pass 0/3000060 on publisher done [07:37:52.010](0.842s) ok 7 - logical slots have synced as true on standby [07:37:52.140](0.130s) ok 8 - synchronized slot has been dropped ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled Waiting for replication conn standby1's replay_lsn to pass 0/40000F8 on publisher done [07:37:53.007](0.867s) ok 9 - synchronized slot has been invalidated ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled #### Begin standard error psql::2: NOTICE: dropped replication slot "lsub1_slot" on publisher psql::3: NOTICE: created replication slot "lsub1_slot" on publisher #### End standard error Waiting for replication conn regress_mysub1's replay_lsn to pass 0/4000130 on publisher done Waiting for replication conn standby1's replay_lsn to pass 0/4000130 on publisher done [07:37:53.176](0.169s) ok 10 - logical slot is re-synced ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled [07:37:53.201](0.025s) ok 11 - logical decoding is not allowed on synced slot [07:37:53.213](0.012s) ok 12 - synced slot on standby cannot be altered [07:37:53.225](0.012s) ok 13 - synced slot on standby cannot be dropped ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled [07:37:53.259](0.033s) not ok 14 - cannot sync slots if dbname is not specified in primary_conninfo [07:37:53.259](0.000s) # Failed test 'cannot sync slots if dbname is not specified in primary_conninfo' # at /tmp/cirrus-ci-build/src/test/recovery/t/040_standby_failover_slots_sync.pl line 325. ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled # Taking pg_basebackup backup2 from node "standby1" # Running: pg_basebackup -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/backup/backup2 -h /tmp/t9VPlUiE88 -p 50632 --checkpoint fast --no-sync # Backup finished # Checking port 50633 # Found port 50633 Name: cascading_standby Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/archives Connection string: port=50633 host=/tmp/t9VPlUiE88 Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_cascading_standby.log # Initializing node "cascading_standby" from backup "backup2" of node "standby1" ### Enabling streaming replication for node "cascading_standby" ### Enabling WAL restore for node "cascading_standby" (cascading_sb_slot,) ### Starting node "cascading_standby" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_cascading_standby.log -o --cluster-name=cascading_standby start waiting for server to start.... done server started # Postmaster PID for node "cascading_standby" is 15277 [07:37:53.710](0.451s) ok 15 - cannot sync slots to a cascading standby server ### Stopping node "cascading_standby" using mode fast # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_cascading_standby_data/pgdata -m fast stop waiting for server to shut down.... done server stopped # No postmaster PID for node "cascading_standby" psql::1: WARNING: databases created by regression test cases should have names including "regression" Waiting for replication conn standby1's replay_lsn to pass 0/4441888 on publisher done ### Stopping node "standby1" using mode fast # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -m fast stop waiting for server to shut down.... done server stopped # No postmaster PID for node "standby1" #### Begin standard error psql::3: WARNING: roles created by regression test cases should have names starting with "regress_" #### End standard error ### Starting node "standby1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log -o --cluster-name=standby1 start waiting for server to start.... done server started # Postmaster PID for node "standby1" is 15443 ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled ### Reloading node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata reload server signaled Waiting for all subscriptions in "subscriber1" to synchronize data done [07:37:54.898](1.188s) ok 16 - restart_lsn and confirmed_flush_lsn of slot lsub1_slot synced to standby (sb2_slot,) # Taking pg_basebackup backup3 from node "publisher" # Running: pg_basebackup -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/backup/backup3 -h /tmp/t9VPlUiE88 -p 50630 --checkpoint fast --no-sync # Backup finished # Checking port 50634 # Found port 50634 Name: standby2 Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/archives Connection string: port=50634 host=/tmp/t9VPlUiE88 Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby2.log # Initializing node "standby2" from backup "backup3" of node "publisher" ### Enabling streaming replication for node "standby2" ### Enabling WAL restore for node "standby2" ### Starting node "standby2" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby2.log -o --cluster-name=standby2 start waiting for server to start.... done server started # Postmaster PID for node "standby2" is 15678 Waiting for replication conn standby2's replay_lsn to pass 0/6000000 on publisher done ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled # Checking port 50635 # Found port 50635 Name: subscriber2 Data directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata Backup directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/backup Archive directory: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/archives Connection string: port=50635 host=/tmp/t9VPlUiE88 Log file: /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_subscriber2.log [07:37:55.306](0.408s) # initializing database system by copying initdb template # Running: cp -RPp /tmp/cirrus-ci-build/build/tmp_install/initdb-template /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata # Running: /tmp/cirrus-ci-build/build/src/test/regress/pg_regress --config-auth /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata ### Starting node "subscriber2" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_subscriber2.log -o --cluster-name=subscriber2 start waiting for server to start.... done server started # Postmaster PID for node "subscriber2" is 15715 #### Begin standard error psql::3: NOTICE: created replication slot "lsub2_slot" on publisher #### End standard error Waiting for all subscriptions in "subscriber2" to synchronize data done ### Stopping node "standby1" using mode fast # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -m fast stop waiting for server to shut down.... done server stopped # No postmaster PID for node "standby1" Waiting for replication conn standby2's replay_lsn to pass 0/6000870 on publisher done [07:37:55.797](0.491s) ok 17 - standby2 gets data from primary Waiting for replication conn regress_mysub2's replay_lsn to pass 0/6000870 on publisher done [07:37:55.866](0.069s) ok 18 - subscriber2 gets data from primary [07:37:55.879](0.013s) ok 19 - subscriber1 doesn't get data from primary until standby1 acknowledges changes ### Starting node "standby1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log -o --cluster-name=standby1 start waiting for server to start.... done server started # Postmaster PID for node "standby1" is 15813 Waiting for replication conn standby1's replay_lsn to pass 0/6000870 on publisher done [07:37:56.033](0.154s) ok 20 - standby1 gets data from primary Waiting for replication conn regress_mysub1's replay_lsn to pass 0/6000870 on publisher done [07:37:56.089](0.056s) ok 21 - subscriber1 gets data from primary after standby1 acknowledges changes Waiting for replication conn regress_mysub1's replay_lsn to pass 0/6004948 on publisher done ### Stopping node "standby1" using mode fast # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -m fast stop waiting for server to shut down.... done server stopped # No postmaster PID for node "standby1" ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled [07:37:57.509](1.419s) ok 22 - subscriber1 doesn't get data as the sb1_slot doesn't catch up ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled Waiting for replication conn regress_mysub1's replay_lsn to pass 0/6004F10 on publisher done [07:37:57.736](0.227s) ok 23 - subscriber1 gets data from primary after standby1 is removed from the standby_slot_names list ### Reloading node "publisher" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata reload server signaled ### Starting node "standby1" # Running: pg_ctl -w -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log -o --cluster-name=standby1 start waiting for server to start.... done server started # Postmaster PID for node "standby1" is 16204 Waiting for replication conn standby1's replay_lsn to pass 0/6004F10 on publisher done ### Promoting node "standby1" # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -l /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/log/040_standby_failover_slots_sync_standby1.log promote waiting for server to promote.... done server promoted [07:37:58.706](0.970s) ok 24 - synced slot retained on the new primary Waiting for replication conn regress_mysub1's replay_lsn to pass 0/6007400 on standby1 done [07:37:58.969](0.263s) ok 25 - data replicated from the new primary [07:37:58.969](0.000s) 1..25 ### Stopping node "publisher" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_publisher_data/pgdata -m immediate stop waiting for server to shut down.... done server stopped # No postmaster PID for node "publisher" ### Stopping node "subscriber1" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber1_data/pgdata -m immediate stop waiting for server to shut down... done server stopped # No postmaster PID for node "subscriber1" ### Stopping node "standby1" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby1_data/pgdata -m immediate stop waiting for server to shut down.... done server stopped # No postmaster PID for node "standby1" ### Stopping node "standby2" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_standby2_data/pgdata -m immediate stop waiting for server to shut down.... done server stopped # No postmaster PID for node "standby2" ### Stopping node "subscriber2" using mode immediate # Running: pg_ctl -D /tmp/cirrus-ci-build/build/testrun/recovery/040_standby_failover_slots_sync/data/t_040_standby_failover_slots_sync_subscriber2_data/pgdata -m immediate stop waiting for server to shut down.... done server stopped # No postmaster PID for node "subscriber2" [07:37:59.462](0.493s) # Looks like you failed 1 test of 25.