more css features (330d8d0f) · Commits · jan.koester / libhtmlpp

src/CMakeLists.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -6,6 +6,7 @@ include(GenerateExportHeader)
		set(headers
		css.h
		html.h
		htmlcss.h
		request.h
		utils.h
		exception.h
		@@ -14,6 +15,7 @@ set(headers
		set(libhtmlSrcs
		css.cpp
		html.cpp
		htmlcss.cpp
		request.cpp
		exception.cpp
		)

src/css.cpp

+244 −1

Original line number	Diff line number	Diff line
		@@ -45,6 +45,124 @@ namespace {
		return s.substr(start, end - start);
		}

		// Strips a trailing "!important" (case-insensitive, any whitespace
		// before the "!") from a CSS declaration value, returning whether it was
		// present. CSSDeclaration's parser captures "!important" as ordinary
		// trailing text with no special handling, so a caller comparing/storing
		// plain property values needs this to see past it.
		bool stripImportant(std::string &value) {
		size_t bang = value.rfind('!');
		if (bang == std::string::npos) return false;

		size_t start = value.find_first_not_of(" \t\n\r", bang + 1);
		size_t end = value.find_last_not_of(" \t\n\r");
		if (start == std::string::npos \|\| end == std::string::npos \|\| start > end) return false;

		std::string tail = value.substr(start, end - start + 1);
		std::transform(tail.begin(), tail.end(), tail.begin(),
		[](unsigned char c) { return std::tolower(c); });
		if (tail != "important") return false;

		size_t valEnd = bang;
		while (valEnd > 0 && isWhitespace(value[valEnd - 1])) --valEnd;
		value.resize(valEnd);
		return true;
		}

		// A combinator-stripped compound selector like "div.card.featured#hero"
		// broken into its optional bare tag (lowercased), every ".class" token,
		// and an optional "#id" token.
		struct CompoundParts {
		std::string tag;
		std::vector<std::string> classes;
		std::string id;
		};

		CompoundParts parseCompoundSelector(const std::string &matchSel) {
		CompoundParts result;
		size_t pos = 0;
		while (pos < matchSel.size()) {
		char c = matchSel[pos];
		if (c == '.' \|\| c == '#') {
		size_t next = matchSel.find_first_of(".#", pos + 1);
		size_t tokenEnd = (next == std::string::npos) ? matchSel.size() : next;
		std::string token = matchSel.substr(pos + 1, tokenEnd - (pos + 1));
		if (c == '.') result.classes.push_back(token);
		else result.id = token;
		pos = tokenEnd;
		} else {
		size_t next = matchSel.find_first_of(".#", pos);
		size_t tokenEnd = (next == std::string::npos) ? matchSel.size() : next;
		result.tag = matchSel.substr(pos, tokenEnd - pos);
		pos = tokenEnd;
		}
		}
		std::transform(result.tag.begin(), result.tag.end(), result.tag.begin(),
		[](unsigned char c) { return std::tolower(c); });
		return result;
		}

		// A selector matches only if EVERY part it specifies is present on this
		// element -- in particular every class listed, not just one. A bare tag
		// left over after stripping a combinator's ancestor part (e.g.
		// "[data-atom=header]>div" reduces to bare "div") can't be verified --
		// this matcher only ever sees the CURRENT element's own tag/class/id,
		// never its ancestor chain -- so treating it as a match would apply the
		// rule to literally every element of that tag in the whole document
		// (confirmed against a real scraped page where exactly this pattern
		// matched ~24,000 times and blew up an HTML-import's output to 36MB). A
		// class/id qualifier alongside the tag (e.g. "div.foo") is still
		// specific enough to check.
		bool compoundMatches(const CompoundParts &compound, bool hadCombinator,
		const std::string &tag, const std::vector<std::string> &classes,
		const std::string &id)
		{
		bool bareTagAfterCombinator = hadCombinator &&
		compound.classes.empty() && compound.id.empty() && !compound.tag.empty();
		bool specifiedSomething = !compound.tag.empty() \|\| !compound.classes.empty() \|\| !compound.id.empty();
		if (bareTagAfterCombinator \|\| !specifiedSomething) return false;

		bool tagOk = compound.tag.empty() \|\| compound.tag == tag;
		bool idOk = compound.id.empty() \|\| compound.id == id;
		bool classesOk = true;
		for (const auto &cls : compound.classes) {
		if (std::find(classes.begin(), classes.end(), cls) == classes.end()) {
		classesOk = false;
		break;
		}
		}
		return tagOk && idOk && classesOk;
		}

		// Strips a trailing combinator (space/">"/"+"/"~") from `singleSel`,
		// returning the trailing compound selector text and whether a
		// combinator was actually present. Only the trailing compound is ever
		// checked -- this matcher never verifies ancestor/parent context.
		std::string stripCombinator(const std::string &singleSel, bool &hadCombinator) {
		std::string matchSel = singleSel;
		hadCombinator = false;
		size_t lastSep = matchSel.find_last_of(" >+~");
		if (lastSep != std::string::npos) {
		size_t mstart = matchSel.find_first_not_of(" >+~", lastSep);
		if (mstart != std::string::npos) {
		matchSel = matchSel.substr(mstart);
		hadCombinator = true;
		}
		}
		return matchSel;
		}

		// Selector syntax this matcher can't safely evaluate -- attribute
		// selectors and pseudo-classes/elements describe context (an ancestor's
		// attribute) or state (":hover") this matcher has no way to check, and
		// "*" would blanket-match every element. Guessing wrong here is worse
		// than not matching at all.
		bool hasUnsupportedSelectorSyntax(const std::string &matchSel) {
		return matchSel.find('[') != std::string::npos \|\|
		matchSel.find(':') != std::string::npos \|\|
		matchSel == "*";
		}

		}

		// --- CSSProperty ---
		@@ -426,3 +544,128 @@ libhtmlpp::CSSDeclaration libhtmlpp::CSSStyleSheet::parseInlineStyle(const std::
		return decl;
		}

		bool libhtmlpp::CSSStyleSheet::approximateSelectorMatch(const std::string &selector,
		const std::string &tag,
		const std::vector<std::string> &classes,
		const std::string &id)
		{
		std::string tagLower = tag;
		std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(),
		[](unsigned char c) { return std::tolower(c); });

		std::istringstream selStream(selector);
		std::string singleSel;
		while (std::getline(selStream, singleSel, ',')) {
		size_t start = singleSel.find_first_not_of(" \t\n\r");
		if (start == std::string::npos) continue;
		size_t end = singleSel.find_last_not_of(" \t\n\r");
		singleSel = singleSel.substr(start, end - start + 1);

		bool hadCombinator = false;
		std::string matchSel = stripCombinator(singleSel, hadCombinator);
		if (hasUnsupportedSelectorSyntax(matchSel)) continue;

		CompoundParts compound = parseCompoundSelector(matchSel);
		if (compoundMatches(compound, hadCombinator, tagLower, classes, id)) return true;
		}
		return false;
		}

		void libhtmlpp::CSSStyleSheet::collectApproximateMatches(
		const std::string &tag,
		const std::string &cssClass,
		const std::string &id,
		std::map<std::string,std::string> &props,
		std::string &mediaRules,
		std::set<std::string> &seenMediaBlocks) const
		{
		std::vector<std::string> classes;
		if (!cssClass.empty()) {
		std::istringstream iss(cssClass);
		std::string cls;
		while (iss >> cls) classes.push_back(cls);
		}

		// Property names already present in @p props when we're called (e.g.
		// the element's own inline style, set by the caller before calling this)
		// outrank a plain stylesheet rule, unless that rule is "!important" --
		// matching real cascade precedence.
		std::set<std::string> inlineKeys;
		for (const auto &kv : props) inlineKeys.insert(kv.first);
		std::set<std::string> importantKeys;

		for (const auto &rule : _Rules) {
		const std::string &sel = rule.getSelector();
		if (sel.empty()) continue;

		bool isAtRule = sel[0] == '@';
		std::string atWrapper;
		std::string innerSel;

		if (isAtRule) {
		size_t parenDepth = 0;
		size_t splitPos = std::string::npos;
		for (size_t i = 0; i < sel.size(); ++i) {
		if (sel[i] == '(') ++parenDepth;
		else if (sel[i] == ')') {
		if (parenDepth > 0) --parenDepth;
		if (parenDepth == 0) { splitPos = i + 1; break; }
		}
		}
		if (splitPos != std::string::npos && splitPos < sel.size()) {
		atWrapper = sel.substr(0, splitPos);
		size_t innerStart = sel.find_first_not_of(" \t\n\r", splitPos);
		if (innerStart != std::string::npos) innerSel = sel.substr(innerStart);
		}
		if (innerSel.empty()) continue;
		}

		const std::string &matchTarget = isAtRule ? innerSel : sel;

		std::istringstream selStream(matchTarget);
		std::string singleSel;
		while (std::getline(selStream, singleSel, ',')) {
		size_t start = singleSel.find_first_not_of(" \t\n\r");
		if (start == std::string::npos) continue;
		size_t end = singleSel.find_last_not_of(" \t\n\r");
		singleSel = singleSel.substr(start, end - start + 1);

		if (!approximateSelectorMatch(singleSel, tag, classes, id)) continue;

		if (isAtRule) {
		// The same @media block, once present anywhere in the final
		// output, applies document-wide regardless of which element
		// it's attached to -- so including it more than once across
		// many matching elements is pure bloat, not a correctness
		// requirement.
		std::string block = atWrapper + " { " + singleSel + " { ";
		for (const auto &prop : rule.getDeclaration().getProperties()) {
		block += prop.getName() + ": " + prop.getValue() + "; ";
		}
		block += "} } ";
		if (seenMediaBlocks.insert(block).second) {
		mediaRules += block;
		}
		} else {
		for (const auto &prop : rule.getDeclaration().getProperties()) {
		std::string value = prop.getValue();
		bool isImportant = stripImportant(value);
		const std::string &key = prop.getName();

		// Inline style and an already-!important value both
		// outrank a later plain rule; a later rule of equal
		// priority (both plain, or both !important) wins --
		// approximating the real cascade's "last rule of
		// equal-or-higher precedence wins" without computing
		// full selector specificity.
		if (inlineKeys.count(key) && !isImportant) continue;
		if (importantKeys.count(key) && !isImportant) continue;

		props[key] = value;
		if (isImportant) importantKeys.insert(key);
		}
		}
		}
		}
		}

src/css.h

+55 −0

Original line number	Diff line number	Diff line
		@@ -29,6 +29,8 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

		#include <string>
		#include <vector>
		#include <map>
		#include <set>
		#include <memory>

		namespace libhtmlpp {
		@@ -122,6 +124,59 @@ namespace libhtmlpp {

		static CSSDeclaration parseInlineStyle(const std::string &style);

		/** Conservative, NOT spec-complete selector match: @p selector (a
		* single selector, or a comma-separated list of them -- matches if
		* ANY branch matches) is matched against a single element
		* identified by @p tag/@p classes/@p id only -- there is no
		* ancestor/parent context available, so
		* any selector this can't safely evaluate without seeing more than
		* that one element is REJECTED (returns false) rather than guessed.
		* Rejected outright: attribute selectors ("[href]"), pseudo-classes
		* and pseudo-elements (":hover", "::before"), the universal selector
		* ("*"), and -- after a combinator (space/">"/"+"/"~") is stripped
		* down to its trailing compound selector -- a bare tag left with no
		* class/id qualifier of its own (e.g. "[data-x]>div" reduces to a
		* bare "div", which would otherwise match every element of that tag
		* in the whole document; a qualified compound like "div.foo" is
		* still specific enough to check). A compound selector like
		* "div.card.featured#hero" matches only if the tag (when given),
		* every class listed (all of them, not just one), and the id (when
		* given) are all present on the element. This is intentionally an
		* approximation of real CSS selector matching, not an
		* implementation of it -- there is no specificity calculation and
		* no combinator/ancestor verification -- built to be safe against
		* false positives on real-world scraped markup rather than
		* complete; treat a `false` result as "not proven to match", not
		* "definitely doesn't". */
		static bool approximateSelectorMatch(const std::string &selector,
		const std::string &tag,
		const std::vector<std::string> &classes,
		const std::string &id);

		/** Runs every rule in this sheet through approximateSelectorMatch
		* against the element identified by @p tag/@p cssClass (a
		* whitespace-separated class list)/@p id, folding matching
		* declarations into @p props under an approximation of the real CSS
		* cascade: a property already present in @p props when this is
		* called (e.g. the element's own inline style, set by the caller
		* before calling this) outranks a later plain rule unless that
		* rule's value carries "!important" (stripped from the stored
		* value either way); among this sheet's own rules, a later rule
		* beats an earlier one at equal priority. This is "last rule of
		* equal-or-higher precedence wins", not full CSS specificity.
		* Each matching `@media`/other at-rule block's raw text is appended
		* to @p mediaRules, skipped if its exact text is already present in
		* @p seenMediaBlocks (both are caller-owned, so a caller processing
		* many elements from one document/sheet can share one instance of
		* each across all of them and avoid repeating an identical block
		* once per matching element). */
		void collectApproximateMatches(const std::string &tag,
		const std::string &cssClass,
		const std::string &id,
		std::map<std::string,std::string> &props,
		std::string &mediaRules,
		std::set<std::string> &seenMediaBlocks) const;

		private:
		void _skipWhitespace(const std::string &input, size_t &pos) const;
		void _skipComment(const std::string &input, size_t &pos) const;

src/htmlcss.cpp

0 → 100644

+135 −0

Original line number	Diff line number	Diff line
		/*******************************************************************************
		Copyright (c) 2021, Jan Koester jan.koester@gmx.net
		All rights reserved.

		Redistribution and use in source and binary forms, with or without
		modification, are permitted provided that the following conditions are met:
		* Redistributions of source code must retain the above copyright
		notice, this list of conditions and the following disclaimer.
		* Redistributions in binary form must reproduce the above copyright
		notice, this list of conditions and the following disclaimer in the
		documentation and/or other materials provided with the distribution.
		* Neither the name of the <organization> nor the
		names of its contributors may be used to endorse or promote products
		derived from this software without specific prior written permission.

		THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
		ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
		WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
		DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
		DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
		(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
		LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
		ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
		(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
		SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
		*******************************************************************************/

		#include "htmlcss.h"

		#include <algorithm>
		#include <vector>

		namespace {

		std::string lowerCopy(const std::string &s) {
		std::string t = s;
		std::transform(t.begin(), t.end(), t.begin(),
		[](unsigned char c) { return std::tolower(c); });
		return t;
		}

		// Strips a trailing "!important" (case-insensitive, any whitespace
		// before the "!") from a CSS declaration value -- see the identical
		// helper in css.cpp for why this matters; duplicated here (rather than
		// exposed from css.cpp) since it's a small, self-contained detail of
		// normalizing a single value string, not selector/cascade logic.
		void stripImportant(std::string &value) {
		size_t bang = value.rfind('!');
		if (bang == std::string::npos) return;

		size_t start = value.find_first_not_of(" \t\n\r", bang + 1);
		size_t end = value.find_last_not_of(" \t\n\r");
		if (start == std::string::npos \|\| end == std::string::npos \|\| start > end) return;

		std::string tail = value.substr(start, end - start + 1);
		std::transform(tail.begin(), tail.end(), tail.begin(),
		[](unsigned char c) { return std::tolower(c); });
		if (tail != "important") return;

		size_t valEnd = bang;
		while (valEnd > 0 && std::isspace(static_cast<unsigned char>(value[valEnd - 1]))) --valEnd;
		value.resize(valEnd);
		}

		}

		void libhtmlpp::collectStyleBlocks(CSSStyleSheet &sheet, Element &root)
		{
		// Children discovered while walking a frame's sibling chain are
		// collected here and pushed in reverse at the end, so <style> blocks
		// end up appended to `sheet` in true document order (matters once two
		// rules for the same property fall back to "later one wins").
		std::vector<Element*> searchStack;
		searchStack.push_back(&root);

		while (!searchStack.empty()) {
		Element *cur = searchStack.back();
		searchStack.pop_back();

		std::vector<Element*> childFrames;

		while (cur) {
		int type = cur->getType();
		if (type == HtmlEl \|\| type == SvgEL \|\| type == TextAreaEL) {
		HtmlElement hel = static_cast<HtmlElement>(cur);
		if (lowerCopy(hel->getTagname()) == "style") {
		std::string cssText;
		Element *child = hel->firstChild();
		while (child) {
		if (child->getType() == TextEl) {
		cssText += static_cast<TextElement*>(child)->getText();
		}
		child = child->nextElement();
		}
		if (!cssText.empty()) {
		CSSStyleSheet partial;
		partial.parse(cssText);
		for (const auto &rule : partial.getRules()) {
		sheet.addRule(rule);
		}
		}
		} else {
		Element *child = hel->firstChild();
		if (child) childFrames.push_back(child);
		}
		}
		cur = cur->nextElement();
		}

		for (auto it = childFrames.rbegin(); it != childFrames.rend(); ++it) {
		searchStack.push_back(*it);
		}
		}
		}

		libhtmlpp::CSSRuleResult libhtmlpp::getCSSRules(HtmlElement &target,
		const CSSStyleSheet &sheet,
		std::set<std::string> &seenMediaBlocks)
		{
		CSSRuleResult result;

		CSSDeclaration inlineDecl = CSSStyleSheet::parseInlineStyle(target.getAtributte("style"));
		for (const auto &prop : inlineDecl.getProperties()) {
		std::string value = prop.getValue();
		stripImportant(value);
		result.properties[prop.getName()] = value;
		}

		std::string tag = lowerCopy(target.getTagname());
		std::string cssClass = target.getAtributte("class");
		std::string id = target.getAtributte("id");

		sheet.collectApproximateMatches(tag, cssClass, id, result.properties, result.mediaRules, seenMediaBlocks);
		return result;
		}

src/htmlcss.h

0 → 100644

+83 −0

File added.

Preview size limit exceeded, changes collapsed.