Skip to content

Flaky raft_metadata_cluster_smoke_test #3914

@tillrohrmann

Description

@tillrohrmann

The test case raft_metadata_cluster_smoke_test failed with:

Value of: client.put(key.clone(), &new_value,
    Precondition::MatchesVersion(value_version.next()),).await
    Expected: is an error which is WriteError::FailedPrecondition(_)
    Actual: Err(
        Other(
            RetryableError(
                StatusError {
                    address: Uds(
                        "/home/runner/work/_temp/lcr/raft_metadata_cluster_smoke_test/node-1/node.sock",
                    ),
                    status: status: 'The service is currently unavailable', self: "lost leadership", metadata: {"content-type": "application/grpc", "date": "Mon, 20 Oct 2025 10:38:43 GMT"},
                },
            ),
        ),
    ),
      which is an error which is not WriteError::FailedPrecondition(_)
      at server/tests/raft_metadata_cluster.rs:97:5

    stack backtrace:
       0:     0x562f7b3b1e72 - <std::sys::backtrace::BacktraceLock::print::DisplayBacktrace as core::fmt::Display>::fmt::hdcfcb6d4c8489523
       1:     0x562f7b3e0953 - core::fmt::write::h8a494366950f23bb
       2:     0x562f7b3ad7f3 - std::io::Write::write_fmt::h6556609fca33d0b1
       3:     0x562f7b3b1cc2 - std::sys::backtrace::BacktraceLock::print::hb2a626a81e06b2dc
       4:     0x562f7b3b3da9 - std::panicking::default_hook::{{closure}}::h4f78485264f12d10
       5:     0x562f7b3b3b8b - std::panicking::default_hook::h2c66fc99e962531d
       6:     0x562f7911af89 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h1248e0675e20cd43
       7:     0x562f791705b6 - raft_metadata_cluster::raft_metadata_cluster_smoke_test::{{closure}}::hbc25b15a7d819523
       8:     0x562f7b3b487e - std::panicking::rust_panic_with_hook::h33ac55f64bbd807d
       9:     0x562f7b3b454a - std::panicking::begin_panic_handler::{{closure}}::h30e7cb89678a57fe
      10:     0x562f7b3b2379 - std::sys::backtrace::__rust_end_short_backtrace::hed60f27456c16ced
      11:     0x562f7b3b41dd - __rustc[de2ca18b4c54d5b8]::rust_begin_unwind
      12:     0x562f7b3ddb00 - core::panicking::panic_fmt::h62f63d096dd276af
      13:     0x562f7916efb7 - raft_metadata_cluster::raft_metadata_cluster_smoke_test::{{closure}}::hfb39846546fea974
      14:     0x562f790f63b8 - <core::pin::Pin<P> as core::future::future::Future>::poll::h0fbd352636c5c8ff
      15:     0x562f7910b54a - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h9e7718db42a661a7
      16:     0x562f79111af0 - tokio::task::task_local::LocalKey<T>::scope_inner::h631a597300c13ae6
      17:     0x562f7910a4d0 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hb5b155383487b362
      18:     0x562f7910b961 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::hc761d20585902d30
      19:     0x562f79113590 - tokio::task::task_local::LocalKey<T>::scope_inner::hd60fffd424e42c64
      20:     0x562f7910a3a0 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hb05766db748d9f30
      21:     0x562f7910aa71 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::{{closure}}::h287b728e11915a02
      22:     0x562f79111590 - tokio::task::task_local::LocalKey<T>::scope_inner::h131b5349913bca88
      23:     0x562f7910a600 - <tokio::task::task_local::TaskLocalFuture<T,F> as core::future::future::Future>::poll::hc329ed6eec959f4a
      24:     0x562f7910c7ce - <restate_core::task_center::extensions::WithTaskCenter<T> as core::future::future::Future>::poll::h66ae5d9534d87696
      25:     0x562f791bbaaf - <tokio::runtime::task::trace::Root<T> as core::future::future::Future>::poll::h0637fb71fe5251b6
      26:     0x562f7913cef2 - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::haad4793526414094
      27:     0x562f790f6427 - <core::pin::Pin<P> as core::future::future::Future>::poll::h129da6c3ecfda106
      28:     0x562f791350c7 - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}}::hf7adc8b39a7b11a2
      29:     0x562f79134b43 - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::h6cf21394410d321d
      30:     0x562f7912ea6d - tokio::runtime::scheduler::current_thread::Context::enter::h3292d1c059cc722c
      31:     0x562f79131b85 - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::h0a64cb4466b9cedd
      32:     0x562f791316fb - tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}::ha0d9b21f5312bcb3
      33:     0x562f7916a730 - tokio::runtime::context::scoped::Scoped<T>::set::hc6e479912dadd1dd
      34:     0x562f791fc3f3 - tokio::runtime::context::set_scheduler::{{closure}}::h379778e4ae55b097
      35:     0x562f791f7da3 - std::thread::local::LocalKey<T>::try_with::h63dda505f3f7ef74
      36:     0x562f791f5ad7 - std::thread::local::LocalKey<T>::with::hb6fa6a6760c24eb3
      37:     0x562f791fc30b - tokio::runtime::context::set_scheduler::h5565a7bfd07cc1aa
      38:     0x562f7913146e - tokio::runtime::scheduler::current_thread::CoreGuard::enter::he6500ad900a6cb3b
      39:     0x562f791317ec - tokio::runtime::scheduler::current_thread::CoreGuard::block_on::h486010417b41e27c
      40:     0x562f7912c122 - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}}::hd6d977bc2eb05f3d
      41:     0x562f791af979 - tokio::runtime::context::runtime::enter_runtime::h57222da96a4823dd
      42:     0x562f7912b75e - tokio::runtime::scheduler::current_thread::CurrentThread::block_on::h829d39af67237a6c
      43:     0x562f791054a3 - tokio::runtime::runtime::Runtime::block_on_inner::hff719ce259dbf590
      44:     0x562f791059ea - tokio::runtime::runtime::Runtime::block_on::hb4631a4d84d4a066
      45:     0x562f7916d248 - raft_metadata_cluster::raft_metadata_cluster_smoke_test::h984e4036c36ea101
      46:     0x562f7916ce28 - raft_metadata_cluster::raft_metadata_cluster_smoke_test::{{closure}}::h69665e15a4c8cdfc
      47:     0x562f790ac346 - core::ops::function::FnOnce::call_once::h08617c893e648d20
      48:     0x562f7926a01b - test::__rust_begin_short_backtrace::ha52ab26e77157f03
      49:     0x562f79269155 - test::run_test::{{closure}}::h0364ba59bf23f652
      50:     0x562f7922cd54 - std::sys::backtrace::__rust_begin_short_backtrace::h177cb952fc54ad2c
      51:     0x562f7923072a - core::ops::function::FnOnce::call_once{{vtable.shim}}::hddcd7484a13fd793
      52:     0x562f7b3b7ccf - std::sys::pal::unix::thread::Thread::new::thread_start::hb6e99e73da4d28f8
      53:     0x7f1a7ae9caa4 - <unknown>
      54:     0x7f1a7af29c6c - <unknown>
      55:                0x0 - <unknown>

https://github.com/restatedev/restate/actions/runs/18649409294/job/53163695698?pr=3913#step:12:606

Metadata

Metadata

Assignees

Labels

No labels
No labels

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions