test (d819125a) · Commits · tuxist / mediadb

debian/changelog

+10 −0

Original line number	Diff line number	Diff line
		mediadb (20260419+29) unstable; urgency=medium

		* Add `POST /api/cluster/repair` endpoint: scans all group IDs across
		cluster peers, identifies orphaned store metadata blobs by magic bytes,
		re-fetches and reloads all store metadata, and re-replicates the
		repaired index. Recovers stores that disappeared due to index corruption
		or failed syncs.

		-- Jan Koester <jan.koester@tuxist.de> Sun, 19 Apr 2026 16:45:00 +0200

		mediadb (20260419+28) unstable; urgency=critical

		* Fix cluster startup deadlock: initial_sync_ok_ was only set on empty

src/app.cpp

+38 −0

Original line number	Diff line number	Diff line
		@@ -538,6 +538,8 @@ HttpResponse App::handle(const HttpRequest& req) {
		return handle_cluster_scrub(req);
		if (req.method == "POST" && req.path == "/api/cluster/rebalance")
		return handle_cluster_rebalance(req);
		if (req.method == "POST" && req.path == "/api/cluster/repair")
		return handle_cluster_repair(req);
		if (req.method == "GET" && req.path == "/api/groups")
		return handle_list_groups(req);

		@@ -947,6 +949,42 @@ HttpResponse App::handle_cluster_rebalance(const HttpRequest& req) {
		return HttpResponse::json(200, j);
		}

		HttpResponse App::handle_cluster_repair(const HttpRequest& req) {
		if (!auth_.is_authorized(req)) return error_json(401, "unauthorized");

		if (!g_Cluster \|\| !g_Cluster->isRunning()) {
		return error_json(400, "cluster not enabled");
		}

		auto* cluster_backend = dynamic_cast<ClusterMediaBackend*>(&db_);
		if (!cluster_backend) {
		return error_json(500, "backend does not support cluster repair");
		}

		try {
		cluster_backend->repair_index();
		} catch (const std::exception& e) {
		return error_json(500, std::string("repair failed: ") + e.what());
		}

		auto stores = db_.list_stores();
		std::size_t total_albums = 0;
		std::size_t total_media = 0;
		for (const auto& s : stores) {
		auto albums = db_.list_albums(s.id);
		total_albums += albums.size();
		for (const auto& a : albums)
		total_media += db_.list_media(a.id).size();
		}

		json_object* j = json_object_new_object();
		json_object_object_add(j, "message", json_object_new_string("repair completed"));
		json_object_object_add(j, "stores", json_object_new_int(static_cast<int>(stores.size())));
		json_object_object_add(j, "albums", json_object_new_int(static_cast<int>(total_albums)));
		json_object_object_add(j, "media", json_object_new_int(static_cast<int>(total_media)));
		return HttpResponse::json(200, j);
		}

		// ---- store handlers ----

		HttpResponse App::handle_create_store(const HttpRequest& req) {

src/app.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -80,6 +80,7 @@ private:
		HttpResponse handle_cluster_sync(const HttpRequest& req);
		HttpResponse handle_cluster_scrub(const HttpRequest& req);
		HttpResponse handle_cluster_rebalance(const HttpRequest& req);
		HttpResponse handle_cluster_repair(const HttpRequest& req);
		HttpResponse handle_list_groups(const HttpRequest& req);
		HttpResponse handle_list_public_albums(const HttpRequest& req);
		HttpResponse handle_create_store(const HttpRequest& req);

src/backend.cpp

+166 −0

Original line number	Diff line number	Diff line
		@@ -3039,6 +3039,172 @@ void ClusterMediaBackend::sync_from_cluster() {

		BinDb& ClusterMediaBackend::local() { return local_; }

		void ClusterMediaBackend::repair_index() {
		if (!cluster_.isRunning()) {
		std::cerr << "[CLUSTER-REPAIR-INDEX] cluster not running\n";
		return;
		}

		// Step 1: Collect all unique group IDs from every peer
		auto peer_groups = cluster_.list_peer_groups();
		std::unordered_set<uint64_t> all_gids;
		for (const auto& pg : peer_groups) {
		if (!pg.online) continue;
		all_gids.insert(pg.groups.begin(), pg.groups.end());
		}

		std::cerr << "[CLUSTER-REPAIR-INDEX] scanning " << all_gids.size()
		<< " unique group IDs across peers\n";

		// Step 2: Try to retrieve every group ID, check if it's store metadata
		auto& cli = cluster_.getClient();
		if (!cli) {
		std::cerr << "[CLUSTER-REPAIR-INDEX] no parity client available\n";
		return;
		}

		// Known magic bytes for store metadata
		static constexpr char S1[4] = {'M', 'D', 'S', '1'};
		static constexpr char S2[4] = {'M', 'D', 'S', '2'};
		static constexpr char S3[4] = {'M', 'D', 'S', '3'};
		// Known magic bytes for index
		static constexpr char I2[4] = {'M', 'D', 'B', '2'};
		static constexpr char I3[4] = {'M', 'D', 'B', '3'};
		static constexpr char I4[4] = {'M', 'D', 'B', '4'};

		auto is_store_magic = [&](const std::vector<uint8_t>& data) {
		if (data.size() < 4) return false;
		return std::memcmp(data.data(), S1, 4) == 0
		\|\| std::memcmp(data.data(), S2, 4) == 0
		\|\| std::memcmp(data.data(), S3, 4) == 0;
		};

		auto is_index_magic = [&](const std::vector<uint8_t>& data) {
		if (data.size() < 4) return false;
		return std::memcmp(data.data(), I2, 4) == 0
		\|\| std::memcmp(data.data(), I3, 4) == 0
		\|\| std::memcmp(data.data(), I4, 4) == 0;
		};

		// Skip group IDs we already know (index, stores, media)
		uint64_t index_gid = cluster_group_id("index");

		std::unordered_set<uint64_t> known_gids;
		known_gids.insert(index_gid);
		{
		std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
		for (const auto& sid : local_.store_ids()) {
		known_gids.insert(cluster_group_id("store:" + sid));
		}
		for (const auto& mid : local_.media_ids()) {
		known_gids.insert(cluster_group_id("media:" + mid));
		}
		}

		// First, try to load the index from the cluster (may contain store IDs we don't know locally)
		std::vector<uint8_t> index_data;
		bool index_ok = cluster_.fetch("index", index_data);
		if (index_ok && !index_data.empty() && is_index_magic(index_data)) {
		std::cerr << "[CLUSTER-REPAIR-INDEX] found valid index (" << index_data.size() << " bytes), loading\n";
		std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
		local_.load_index_from_buffer(index_data);
		}

		// Now try unknown group IDs — these might be orphaned stores
		int found_stores = 0;
		int scanned = 0;
		for (uint64_t gid : all_gids) {
		if (known_gids.count(gid)) continue;
		++scanned;

		std::vector<uint8_t> data;
		try {
		data = cli->retrieve(gid);
		} catch (...) {
		continue;
		}
		if (data.empty()) continue;

		if (is_store_magic(data)) {
		// Found orphaned store metadata! We need to figure out its store_id.
		// Parse the store buffer to extract album IDs, then find a store_id
		// from the album records.
		std::cerr << "[CLUSTER-REPAIR-INDEX] found store metadata at gid=" << gid
		<< " (" << data.size() << " bytes)\n";

		// We need to find the store_id that hashes to this gid.
		// Check all locally known store IDs first.
		std::string matched_sid;
		{
		std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
		for (const auto& sid : local_.store_ids()) {
		if (cluster_group_id("store:" + sid) == gid) {
		matched_sid = sid;
		break;
		}
		}
		}

		if (!matched_sid.empty()) {
		std::cerr << "[CLUSTER-REPAIR-INDEX] matched gid=" << gid
		<< " to known store:" << matched_sid << "\n";
		std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
		local_.load_store_metadata_from_buffer(matched_sid, data);
		++found_stores;
		} else {
		std::cerr << "[CLUSTER-REPAIR-INDEX] store at gid=" << gid
		<< " has no matching local store_id (orphan)\n";
		}
		} else if (is_index_magic(data)) {
		std::cerr << "[CLUSTER-REPAIR-INDEX] found additional index at gid=" << gid
		<< " (" << data.size() << " bytes), loading\n";
		std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
		local_.load_index_from_buffer(data);
		}
		}

		// Step 3: With the (possibly updated) index, now try to fetch ALL store metadata
		std::vector<std::string> sids;
		{
		std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
		sids = local_.store_ids();
		}

		int repaired = 0;
		for (const auto& sid : sids) {
		std::vector<uint8_t> store_data;
		bool ok = cluster_.fetch("store:" + sid, store_data);
		if (!ok \|\| store_data.empty()) {
		std::cerr << "[CLUSTER-REPAIR-INDEX] store:" << sid << " fetch failed\n";
		continue;
		}
		std::unique_lock<std::shared_mutex> cguard(cluster_op_mutex_);
		local_.load_store_metadata_from_buffer(sid, store_data);
		++repaired;
		}

		// Step 4: Re-replicate the repaired index
		{
		std::shared_lock<std::shared_mutex> cguard(cluster_op_mutex_);
		auto sids2 = local_.store_ids();
		std::size_t total_albums = 0;
		std::size_t total_media = 0;
		for (const auto& sid : sids2) {
		auto albums = local_.list_albums(sid);
		total_albums += albums.size();
		for (const auto& a : albums)
		total_media += local_.list_media(a.id).size();
		}
		std::cerr << "[CLUSTER-REPAIR-INDEX] result: " << sids2.size() << " stores, "
		<< total_albums << " albums, " << total_media << " media items"
		<< " (scanned=" << scanned << " unknown gids, found_stores=" << found_stores
		<< ", repaired=" << repaired << ")\n";
		}

		initial_sync_ok_.store(true);
		replicate_index();
		}

		void ClusterMediaBackend::replicate_index() {
		if (!cluster_.isRunning()) return;
		auto buf = local_.save_index_to_buffer();

src/backend.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -324,6 +324,7 @@ public:
		// cluster sync
		void start_sync();
		void sync_from_cluster();
		void repair_index();
		BinDb& local();

		private: