external media suppurt html import (5e123618) · Commits · jan.koester / blogi

editor/html/index.html

+17 −0

Changes for editor/html/index.html: 17 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -304,6 +304,23 @@
		<div id="import-html-step1">
		<label class="file-upload-label">📂 <span data-i18n="I18N_UPLOAD_FILE">Datei wählen</span><input type="file" id="import-html-file" accept=".html,.htm,.txt" style="display:none"></label>
		<textarea id="import-html-content" rows="15" placeholder="HTML hier einfügen..."></textarea>
		<div class="set-field">
		<label><input type="checkbox" id="import-html-media-enabled"> <span data-i18n="I18N_IMPORT_HTML_MEDIA_ENABLE">Externe Bilder automatisch in MediaDB importieren</span></label>
		</div>
		<div id="import-html-media-options" style="display:none">
		<div class="set-field">
		<label data-i18n="I18N_CONNECTION">Verbindung</label>
		<select id="import-html-media-conn"></select>
		</div>
		<div class="set-field">
		<label data-i18n="I18N_MEDIA_STORE">Store</label>
		<select id="import-html-media-store" disabled></select>
		</div>
		<div class="set-field">
		<label data-i18n="I18N_MEDIA_ALBUM">Album</label>
		<select id="import-html-media-album" disabled></select>
		</div>
		</div>
		</div>
		<div id="import-html-step2" style="display:none">
		<p class="hint" data-i18n="I18N_IMPORT_HTML_PICK_HINT">Welche Bereiche sollen importiert werden?</p>

editor/html/js/api.js

+2 −1

Changes for editor/html/js/api.js: 2 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -535,11 +535,12 @@ var EditorApi = (function() {
		return request('POST', '/api/document/parse-html', { html: html });
		},

		importHtml: function(html, mode, parentUuid, sections) {
		importHtml: function(html, mode, parentUuid, sections, mediaImport) {
		var data = { html: html };
		if (mode) data.mode = mode;
		if (parentUuid) data.parent_uuid = parentUuid;
		if (sections) data.sections = sections;
		if (mediaImport) data.media_import = mediaImport;
		return request('POST', '/api/document/import-html', data);
		},

editor/html/js/editor.js

+150 −1

Changes for editor/html/js/editor.js: 150 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -531,10 +531,130 @@
		document.getElementById('import-html-content').value = '';
		importHtmlPendingSource = '';
		importHtmlPendingCandidates = [];
		resetImportHtmlMediaOptions();
		showImportHtmlStep(1);
		document.getElementById('import-html-dialog').showModal();
		}

		// --- HTML import dialog: "import external images into MediaDB" options ---

		function resetImportHtmlMediaOptions() {
		document.getElementById('import-html-media-enabled').checked = false;
		document.getElementById('import-html-media-options').style.display = 'none';
		document.getElementById('import-html-media-conn').innerHTML = '';
		var storeSel = document.getElementById('import-html-media-store');
		var albumSel = document.getElementById('import-html-media-album');
		storeSel.innerHTML = '';
		storeSel.disabled = true;
		albumSel.innerHTML = '';
		albumSel.disabled = true;
		}

		function populateImportHtmlMediaConnections() {
		var connSel = document.getElementById('import-html-media-conn');
		connSel.innerHTML = '';
		EditorApi.listConnections().then(function(resp) {
		var conns = resp.connections \|\| [];
		connSel.innerHTML = '';
		for (var i = 0; i < conns.length; i++) {
		var opt = document.createElement('option');
		opt.value = conns[i].id;
		opt.textContent = (conns[i].name \|\| conns[i].id) + (connectedConnections[conns[i].id] ? ' ✓' : '');
		connSel.appendChild(opt);
		}
		if (conns.length) populateImportHtmlMediaStores(connSel.value);
		}).catch(function() {
		connSel.innerHTML = '';
		});
		}

		function populateImportHtmlMediaStores(connId) {
		var storeSel = document.getElementById('import-html-media-store');
		var albumSel = document.getElementById('import-html-media-album');
		storeSel.innerHTML = '';
		storeSel.disabled = true;
		albumSel.innerHTML = '';
		albumSel.disabled = true;
		if (!connId) return;
		EditorApi.mediaListStores(connId).then(function(resp) {
		var stores = resp.stores \|\| [];
		storeSel.innerHTML = '';
		for (var i = 0; i < stores.length; i++) {
		var opt = document.createElement('option');
		opt.value = stores[i];
		opt.textContent = stores[i];
		storeSel.appendChild(opt);
		}
		storeSel.disabled = !stores.length;
		if (stores.length) populateImportHtmlMediaAlbums(connId, storeSel.value);
		}).catch(function() {
		storeSel.innerHTML = '';
		});
		}

		function populateImportHtmlMediaAlbums(connId, storeName) {
		var albumSel = document.getElementById('import-html-media-album');
		albumSel.innerHTML = '';
		albumSel.disabled = true;
		if (!connId \|\| !storeName) return;
		EditorApi.mediaListAlbums(connId).then(function(resp) {
		var albums = (resp.albums \|\| []).filter(function(a) { return (a.store \|\| '') === storeName; });
		albumSel.innerHTML = '';
		for (var i = 0; i < albums.length; i++) {
		var opt = document.createElement('option');
		opt.value = albums[i].id;
		var cnt = (albums[i].media_count !== undefined) ? (' [' + albums[i].media_count + ']') : '';
		opt.textContent = albums[i].name + cnt;
		albumSel.appendChild(opt);
		}
		var newOpt = document.createElement('option');
		newOpt.value = '__new__';
		newOpt.textContent = I18n.t('I18N_MEDIA_ALBUM_NEW', '+ Neues Album…');
		albumSel.appendChild(newOpt);
		albumSel.disabled = false;
		}).catch(function() {
		albumSel.innerHTML = '';
		});
		}

		// Prompts for a name and creates it via the same mediaCreateAlbum call
		// the ZIP-import flow already uses, then selects the new album --
		// resolving album creation client-side, before importHtml is ever
		// called, so the backend only ever receives a concrete album_id.
		function createImportHtmlMediaAlbum(connId, storeName) {
		var name = prompt(I18n.t('I18N_MEDIA_ALBUM_NEW_PROMPT', 'Name des neuen Albums:'), '');
		if (!name \|\| !name.trim()) {
		populateImportHtmlMediaAlbums(connId, storeName);
		return;
		}
		EditorApi.mediaCreateAlbum(connId, storeName, name.trim(), true).then(function(resp) {
		var album = resp.album;
		if (!album \|\| !album.id) {
		populateImportHtmlMediaAlbums(connId, storeName);
		return;
		}
		var albumSel = document.getElementById('import-html-media-album');
		var opt = document.createElement('option');
		opt.value = album.id;
		opt.textContent = album.name \|\| name.trim();
		albumSel.insertBefore(opt, albumSel.lastChild);
		albumSel.value = album.id;
		}).catch(function(err) {
		alert(I18n.t('I18N_ACTION_FAILED', 'Aktion fehlgeschlagen') + ': ' + ((err && err.error) \|\| ''));
		populateImportHtmlMediaAlbums(connId, storeName);
		});
		}

		// null unless the user checked "import into MediaDB" AND resolved a
		// concrete album (not left on the "+ New album..." placeholder).
		function getImportHtmlMediaImportOptions() {
		if (!document.getElementById('import-html-media-enabled').checked) return null;
		var connId = document.getElementById('import-html-media-conn').value;
		var albumId = document.getElementById('import-html-media-album').value;
		if (!connId \|\| !albumId \|\| albumId === '__new__') return null;
		return { conn_id: connId, album_id: albumId };
		}

		function showImportHtmlStep(step) {
		document.getElementById('import-html-step1').style.display = step === 1 ? '' : 'none';
		document.getElementById('import-html-step2').style.display = step === 2 ? '' : 'none';
		@@ -814,6 +934,28 @@
		setImportHtmlCandidatesChecked(false);
		});

		document.getElementById('import-html-media-enabled').addEventListener('change', function() {
		var enabled = this.checked;
		document.getElementById('import-html-media-options').style.display = enabled ? '' : 'none';
		if (enabled) populateImportHtmlMediaConnections();
		});

		document.getElementById('import-html-media-conn').addEventListener('change', function() {
		populateImportHtmlMediaStores(this.value);
		});

		document.getElementById('import-html-media-store').addEventListener('change', function() {
		populateImportHtmlMediaAlbums(document.getElementById('import-html-media-conn').value, this.value);
		});

		document.getElementById('import-html-media-album').addEventListener('change', function() {
		if (this.value === '__new__') {
		createImportHtmlMediaAlbum(
		document.getElementById('import-html-media-conn').value,
		document.getElementById('import-html-media-store').value);
		}
		});

		document.getElementById('btn-import-html-confirm').addEventListener('click', function() {
		var html = importHtmlPendingSource;
		if (!html) return;
		@@ -825,11 +967,18 @@
		if (boxes[i].checked) sections.push(boxes[i].getAttribute('data-path'));
		}
		}
		EditorApi.importHtml(html, null, null, sections).then(function() {
		var mediaImport = getImportHtmlMediaImportOptions();
		EditorApi.importHtml(html, null, null, sections, mediaImport).then(function(resp) {
		document.getElementById('import-html-dialog').close();
		Canvas.clearSelection(); DocumentTree.clearSelection();
		PropertiesPanel.clear();
		refreshDocument();
		var parts = [];
		if (resp.images_imported) parts.push(resp.images_imported + ' Bild(er) in MediaDB importiert');
		if (resp.images_failed) parts.push(resp.images_failed + ' Bild(er) fehlgeschlagen');
		if (resp.css_fetched) parts.push(resp.css_fetched + ' externes Stylesheet geladen');
		if (resp.css_failed) parts.push(resp.css_failed + ' externes Stylesheet fehlgeschlagen');
		if (parts.length) alert(parts.join(', ') + '.');
		}).catch(function(err) {
		alert(I18n.t('I18N_IMPORT_FAILED') + ': ' + (err.error \|\| ''));
		});

editor/src/htmlimport.cpp

+205 −2

Changes for editor/src/htmlimport.cpp: 205 added lines, 2 removed lines.
Original line number	Diff line number	Diff line
		@@ -29,11 +29,170 @@

		#include <algorithm>
		#include <cctype>
		#include <cstring>
		#include <sstream>
		#include <stack>

		#include <httppp/http.h>
		#include <httppp/httpdefinitions.h>

		#include <netdb.h>
		#include <sys/socket.h>
		#include <netinet/in.h>
		#include <arpa/inet.h>

		using namespace blogi::htmlimport;

		namespace {

		// Best-effort SSRF guard for fetchExternalResource: true if @p addr is
		// loopback/RFC1918-private/link-local/unique-local/unspecified/multicast/
		// reserved -- i.e. anything that isn't a normal public unicast address.
		// IPv4-mapped IPv6 addresses (::ffff:a.b.c.d) are unwrapped and re-checked
		// as IPv4 so that representation can't be used to smuggle a blocked address
		// past the IPv6 branch.
		bool isBlockedAddress(const sockaddr *addr)
		{
		if (addr->sa_family == AF_INET) {
		const auto sin = reinterpret_cast<const sockaddr_in>(addr);
		std::uint32_t ip = ntohl(sin->sin_addr.s_addr);
		if ((ip & 0xFF000000u) == 0x7F000000u) return true; // 127.0.0.0/8
		if ((ip & 0xFF000000u) == 0x0A000000u) return true; // 10.0.0.0/8
		if ((ip & 0xFFF00000u) == 0xAC100000u) return true; // 172.16.0.0/12
		if ((ip & 0xFFFF0000u) == 0xC0A80000u) return true; // 192.168.0.0/16
		if ((ip & 0xFFFF0000u) == 0xA9FE0000u) return true; // 169.254.0.0/16
		if ((ip & 0xFF000000u) == 0x00000000u) return true; // 0.0.0.0/8
		if ((ip & 0xFFC00000u) == 0x64400000u) return true; // 100.64.0.0/10 (CGNAT)
		if ((ip & 0xF0000000u) == 0xE0000000u) return true; // 224.0.0.0/4 multicast
		if ((ip & 0xF0000000u) == 0xF0000000u) return true; // 240.0.0.0/4 reserved
		return false;
		}
		if (addr->sa_family == AF_INET6) {
		const auto sin6 = reinterpret_cast<const sockaddr_in6>(addr);
		const unsigned char *b = sin6->sin6_addr.s6_addr;

		static const unsigned char kLoopback[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1};
		static const unsigned char kUnspecified[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
		static const unsigned char kV4MappedPrefix[12] = {0,0,0,0,0,0,0,0,0,0,0xff,0xff};

		if (std::memcmp(b, kLoopback, 16) == 0) return true;
		if (std::memcmp(b, kUnspecified, 16) == 0) return true;
		if (std::memcmp(b, kV4MappedPrefix, 12) == 0) {
		sockaddr_in mapped{};
		mapped.sin_family = AF_INET;
		std::memcpy(&mapped.sin_addr, b + 12, 4);
		return isBlockedAddress(reinterpret_cast<const sockaddr*>(&mapped));
		}
		if (b[0] == 0xfe && (b[1] & 0xc0) == 0x80) return true; // fe80::/10 link-local
		if ((b[0] & 0xfe) == 0xfc) return true; // fc00::/7 unique-local
		if (b[0] == 0xff) return true; // ff00::/8 multicast
		return false;
		}
		return true; // unknown address family -- reject conservatively
		}

		// Resolves @p host and rejects it if ANY resulting address is blocked (see
		// isBlockedAddress). This is a point-in-time check: the HTTP client below
		// resolves the same hostname again itself when it actually connects, so a
		// DNS answer that changes between this check and that connect (classic
		// DNS-rebinding) is not caught here -- closing that fully would need
		// resolved-IP pinning support in libhttppp, which doesn't exist today.
		bool isSafeExternalHost(const std::string &host, std::string &error)
		{
		struct addrinfo hints{};
		hints.ai_family = AF_UNSPEC;
		hints.ai_socktype = SOCK_STREAM;
		struct addrinfo *res = nullptr;

		int rc = getaddrinfo(host.c_str(), nullptr, &hints, &res);
		if (rc != 0 \|\| !res) {
		error = "could not resolve host";
		return false;
		}

		bool safe = true;
		for (struct addrinfo *p = res; p; p = p->ai_next) {
		if (isBlockedAddress(p->ai_addr)) {
		safe = false;
		break;
		}
		}
		freeaddrinfo(res);

		if (!safe) error = "host resolves to a non-public address";
		return safe;
		}

		} // namespace

		bool blogi::htmlimport::fetchExternalResource(const std::string &url,
		size_t maxBytes,
		const std::string &contentTypePrefix,
		std::vector<std::uint8_t> &out,
		std::string &error)
		{
		out.clear();
		error.clear();

		try {
		libhttppp::HttpUrl httpUrl(url);

		int proto = httpUrl.getProtocol();
		if (proto != libhttppp::HttpUrl::HTTP && proto != libhttppp::HttpUrl::HTTPS) {
		error = "unsupported scheme (only http/https allowed)";
		return false;
		}

		if (!isSafeExternalHost(httpUrl.getHost(), error)) {
		return false;
		}

		libhttppp::HttpClient client(httpUrl);
		client.setTimeout(10);
		client.setMaxRedirects(0);

		libhttppp::HttpRequest req;
		req.setRequestType(GETREQUEST);
		req.setRequestURL(httpUrl.getPath());
		req.setHeaderData("host")->push_back(httpUrl.getHost());

		std::vector<char> resp = client.Get(req);

		int status = client.lastStatusCode();
		if (status < 200 \|\| status >= 300) {
		error = "unexpected HTTP status " + std::to_string(status);
		return false;
		}
		if (resp.empty()) {
		error = "empty response";
		return false;
		}
		// Best-effort only: a server that lies about/omits Content-Length
		// (or abuses chunked encoding) can still have made the underlying
		// client buffer more than maxBytes before we ever get to check --
		// see fetchExternalResource's header doc comment.
		if (resp.size() > maxBytes) {
		error = "response too large";
		return false;
		}
		if (!contentTypePrefix.empty()) {
		const std::string &ct = client.lastContentType();
		if (!ct.empty() && ct.compare(0, contentTypePrefix.size(), contentTypePrefix) != 0) {
		error = "unexpected content type: " + ct;
		return false;
		}
		}

		out.assign(resp.begin(), resp.end());
		return true;
		} catch (const std::exception &e) {
		error = e.what();
		} catch (...) {
		error = "request failed";
		}
		return false;
		}

		std::map<std::string,std::string>
		blogi::htmlimport::declarationToMap(
		const libhtmlpp::CSSDeclaration &decl)
		@@ -1005,7 +1164,8 @@ blogi::htmlimport::buildImportTree(const std::string &html)
		bool blogi::htmlimport::convertHtmlToWidgetXml(const std::string &html,
		std::string &outXml,
		std::string &outGlobalCss,
		const std::set<std::string> *selectedPaths)
		const std::set<std::string> *selectedPaths,
		ImportStats *outStats)
		{
		libhtmlpp::HtmlPage page;
		libhtmlpp::HtmlElement index;
		@@ -1035,7 +1195,12 @@ bool blogi::htmlimport::convertHtmlToWidgetXml(const std::string &html,
		return true;
		}

		// Collect all <style> elements and parse CSS
		// Collect all <style> elements (and, up to a small cap, externally
		// fetched <link rel="stylesheet"> sheets) and parse their CSS.
		const int kMaxExternalStylesheets = 10;
		const size_t kMaxStylesheetBytes = 3 * 1024 * 1024;
		int externalCssSeen = 0;

		libhtmlpp::CSSStyleSheet stylesheet;
		std::stack<libhtmlpp::Element*> searchStack;
		searchStack.push(root);
		@@ -1066,6 +1231,44 @@ bool blogi::htmlimport::convertHtmlToWidgetXml(const std::string &html,
		stylesheet.addRule(rule);
		}
		}
		} else if (tagName == "link") {
		std::string rel = hel->getAtributte("rel");
		std::transform(rel.begin(), rel.end(), rel.begin(),
		[](unsigned char c) { return std::tolower(c); });
		std::string href = hel->getAtributte("href");
		bool isAbsolute = href.compare(0, 7, "http://") == 0 \|\|
		href.compare(0, 8, "https://") == 0;

		if (rel.find("stylesheet") != std::string::npos && isAbsolute) {
		if (externalCssSeen >= kMaxExternalStylesheets) {
		if (outStats) outStats->cssFailed++;
		} else {
		externalCssSeen++;
		// Fetch+parse are wrapped here (unlike the inline
		// <style> path above) because externally fetched
		// CSS is far less trustworthy than same-document
		// markup -- a malformed/hostile sheet must never
		// abort the whole import.
		try {
		std::vector<std::uint8_t> bytes;
		std::string fetchError;
		if (fetchExternalResource(href, kMaxStylesheetBytes,
		"text/css", bytes, fetchError)) {
		std::string cssText(bytes.begin(), bytes.end());
		libhtmlpp::CSSStyleSheet partial;
		partial.parse(cssText);
		for (const auto &rule : partial.getRules()) {
		stylesheet.addRule(rule);
		}
		if (outStats) outStats->cssFetched++;
		} else if (outStats) {
		outStats->cssFailed++;
		}
		} catch (...) {
		if (outStats) outStats->cssFailed++;
		}
		}
		}
		}
		if (tagName != "style") {
		libhtmlpp::Element *child = hel->firstChild();

editor/src/htmlimport.h

+27 −1

Changes for editor/src/htmlimport.h: 27 added lines, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@
		#include <map>
		#include <vector>
		#include <set>
		#include <cstdint>

		#include <htmlpp/html.h>
		#include <htmlpp/css.h>
		@@ -39,6 +40,30 @@
		namespace blogi {
		namespace htmlimport {

		/** Downloads @p url (http/https only) with defenses appropriate for a
		* URL that came from user-supplied content (pasted/uploaded HTML), not
		* a trusted/already-connected server -- see fetchExternalResource's
		* definition in htmlimport.cpp for exactly what is and isn't covered
		* (best-effort SSRF host-range check, no redirects, timeout, and a
		* post-fetch size cap that can't prevent a malicious server from
		* streaming more than @p maxBytes before the check runs).
		* @p contentTypePrefix, if non-empty, rejects responses whose
		* Content-Type doesn't start with it (a missing header is tolerated).
		* Returns false and sets @p error on any failure; never throws. */
		bool fetchExternalResource(const std::string &url,
		size_t maxBytes,
		const std::string &contentTypePrefix,
		std::vector<std::uint8_t> &out,
		std::string &error);

		/** Counters describing what convertHtmlToWidgetXml's external-CSS
		* fetching (see the "link" branch in its <style>-collection loop) did,
		* for callers that want to surface a summary to the user. */
		struct ImportStats {
		int cssFetched = 0;
		int cssFailed = 0;
		};

		std::map<std::string,std::string> declarationToMap(
		const libhtmlpp::CSSDeclaration &decl);

		@@ -100,7 +125,8 @@ namespace htmlimport {
		bool convertHtmlToWidgetXml(const std::string &html,
		std::string &outXml,
		std::string &outGlobalCss,
		const std::set<std::string> *selectedPaths = nullptr);
		const std::set<std::string> *selectedPaths = nullptr,
		ImportStats *outStats = nullptr);

		} // namespace htmlimport
		} // namespace blogi