Commit 3a0303b8 authored by jan.koester's avatar jan.koester
Browse files

better delete

parent 6541a85e

debian/changelog

+9 −0
Original line number Diff line number Diff line 
mediadb (20260419+32) unstable; urgency=medium



  * Cluster-wide store deletion via tombstones: deleting a store on one

    node now replicates a tombstone record so all other nodes remove the

    store on their next sync cycle. Previously stores reappeared after

    being deleted on only one node.



 -- Jan Koester <jan.koester@tuxist.de>  Sat, 19 Apr 2026 00:00:00 +0200



mediadb (20260419+31) unstable; urgency=critical



  * Fix critical data corruption during concurrent import + periodic

src/backend.cpp

+69 −0
Original line number Diff line number Diff line
@@ -2511,6 +2511,9 @@ bool ClusterMediaBackend::delete_store(const std::string& id) { 
        cluster_.remove("media:" + mid);

    }

    

    // Record tombstone so other nodes also delete this store on sync

    tombstones_.insert(id);

    replicate_tombstones();

    replicate_index();

    return true;

}
@@ -3016,6 +3019,13 @@ void ClusterMediaBackend::sync_from_cluster() { 
    }

    initial_sync_ok_.store(true);



    // Load tombstones from cluster and delete any stores that were deleted elsewhere

    load_tombstones();

    {

        std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);

        apply_tombstones();

    }



    // Snapshot store IDs under shared lock

    std::vector<std::string> sids;

    {
@@ -3228,6 +3238,65 @@ void ClusterMediaBackend::replicate_store(const std::string& store_id) { 
    cluster_.replicate("store:" + store_id, buf.data(), buf.size());

}



void ClusterMediaBackend::replicate_tombstones() {

    if (!cluster_.isRunning()) return;

    if (tombstones_.empty()) return;

    // Serialise: u32 count, then each id as length-prefixed string

    std::vector<std::uint8_t> buf;

    auto push_u32 = [&](std::uint32_t v) {

        buf.push_back(static_cast<uint8_t>(v & 0xFF));

        buf.push_back(static_cast<uint8_t>((v >> 8) & 0xFF));

        buf.push_back(static_cast<uint8_t>((v >> 16) & 0xFF));

        buf.push_back(static_cast<uint8_t>((v >> 24) & 0xFF));

    };

    push_u32(static_cast<std::uint32_t>(tombstones_.size()));

    for (const auto& id : tombstones_) {

        push_u32(static_cast<std::uint32_t>(id.size()));

        buf.insert(buf.end(), id.begin(), id.end());

    }

    cluster_.replicate("tombstones", buf.data(), buf.size());

}



void ClusterMediaBackend::load_tombstones() {

    if (!cluster_.isRunning()) return;

    try {

        std::vector<uint8_t> buf;

        bool ok = cluster_.fetch("tombstones", buf);

        if (!ok || buf.size() < 4) return;

        size_t pos = 0;

        auto read_u32 = [&]() -> std::uint32_t {

            if (pos + 4 > buf.size()) return 0;

            std::uint32_t v = static_cast<uint8_t>(buf[pos])

                            | (static_cast<uint8_t>(buf[pos+1]) << 8)

                            | (static_cast<uint8_t>(buf[pos+2]) << 16)

                            | (static_cast<uint8_t>(buf[pos+3]) << 24);

            pos += 4;

            return v;

        };

        auto count = read_u32();

        for (std::uint32_t i = 0; i < count && pos < buf.size(); ++i) {

            auto len = read_u32();

            if (pos + len > buf.size()) break;

            std::string id(buf.begin() + pos, buf.begin() + pos + len);

            pos += len;

            tombstones_.insert(std::move(id));

        }

    } catch (...) {

        // No tombstones blob yet — that's fine

    }

}



void ClusterMediaBackend::apply_tombstones() {

    // Remove any stores present in the tombstone list

    for (const auto& id : tombstones_) {

        if (local_.get_store(id)) {

            local_.delete_store(id);

            // Also remove cluster blobs if still present

            try { cluster_.remove("store:" + id); } catch (...) {}

        }

    }

}



void ClusterMediaBackend::repair_replication() {

    if (!cluster_.isRunning()) return;

    if (importing_.load()) return;

src/backend.h

+7 −0
Original line number Diff line number Diff line
@@ -330,6 +330,9 @@ public: 
private:

    void replicate_index();

    void replicate_store(const std::string& store_id);

    void replicate_tombstones();

    void load_tombstones();

    void apply_tombstones();

    std::vector<std::uint8_t> export_db_to_buffer_impl() const;

    void repair_replication();

    void sync_loop();
@@ -346,6 +349,10 @@ private: 


    BlobCache& cache_;



    // Tombstones: store IDs that have been explicitly deleted.

    // Replicated as cluster key "tombstones" so all nodes know about deletions.

    std::set<std::string> tombstones_;



    std::vector<std::uint8_t> cache_fetch_range(const std::string& media_id,

                                                 std::uint64_t offset,

                                                 std::uint64_t length) const;