Commit 43e15a53 authored by jan.koester's avatar jan.koester
Browse files

test

parent b33eba1b

debian/changelog

+11 −0
Original line number Diff line number Diff line 
mediadb (20260419+34) unstable; urgency=high



  * Fix index replication deadlock: repair_replication() no longer requires

    initial_sync_ok_ to re-replicate the index. Previously, if the index

    was only on 1/3 nodes after restart, all nodes were stuck: sync_from_cluster

    couldn't fetch the index, and repair_replication refused to run because

    initial_sync_ok was false. Now the index is always repaired if under-replicated,

    which unblocks the other nodes' sync.



 -- Jan Koester <jan.koester@tuxist.de>  Sat, 19 Apr 2026 00:00:00 +0200



mediadb (20260419+33) unstable; urgency=medium



  * Fix startup hang: start_sync now has a 30-second total time budget.

src/backend.cpp

+17 −6
Original line number Diff line number Diff line
@@ -3308,7 +3308,6 @@ void ClusterMediaBackend::apply_tombstones() { 
void ClusterMediaBackend::repair_replication() {

    if (!cluster_.isRunning()) return;

    if (importing_.load()) return;

    if (!initial_sync_ok_.load()) return;

    size_t total_peers = cluster_.getTotalPeers();

    if (total_peers < 2) return;
@@ -3327,7 +3326,9 @@ void ClusterMediaBackend::repair_replication() { 
        if (pg.online) ++online_count;

    if (online_count < required_shards) return; // not enough nodes online to repair



    // Check index

    // Always check index replication — even before initial_sync_ok_.

    // If this node has local data, it can push the index to the cluster

    // to unblock other nodes that are stuck fetching.

    uint64_t index_gid = cluster_group_id("index");

    int index_shards = 0;

    for (size_t i = 0; i < peer_groups.size(); ++i) {
@@ -3335,9 +3336,17 @@ void ClusterMediaBackend::repair_replication() { 
            ++index_shards;

    }

    if (index_shards < static_cast<int>(required_shards)) {

        replicate_index();

        auto buf = local_.save_index_to_buffer();

        if (!buf.empty()) {

            std::cerr << "[CLUSTER-REPAIR] re-replicating index ("

                      << index_shards << "/" << required_shards << ")\n";

            cluster_.replicate("index", buf.data(), buf.size());

        }

    }



    // Everything below requires a successful initial sync

    if (!initial_sync_ok_.load()) return;



    // Check stores

    for (const auto& sid : local_.store_ids()) {

        uint64_t gid = cluster_group_id("store:" + sid);
@@ -3386,7 +3395,9 @@ void ClusterMediaBackend::sync_loop() { 
        if (importing_.load()) continue;

        try {

            sync_from_cluster();

            // Run repair every 6th cycle (~30s)

            // Run repair every 6th cycle (~30s).

            // repair_replication() can fix index even before initial_sync_ok_

            // so nodes stuck in "fetch failed" can recover.

            if (++cycle >= 6) {

                cycle = 0;

                if (!importing_.load()) {
@@ -3394,9 +3405,9 @@ void ClusterMediaBackend::sync_loop() { 
                }

            }

        } catch (const std::exception& e) {

            DBG_LOG("[CLUSTER-SYNC] periodic sync failed: " << e.what() << "\n");

            std::cerr << "[CLUSTER-SYNC] periodic sync failed: " << e.what() << "\n";

        } catch (...) {

            DBG_LOG("[CLUSTER-SYNC] periodic sync failed (unknown)\n");

            std::cerr << "[CLUSTER-SYNC] periodic sync failed (unknown)\n";

        }

    }

}