diff --git a/.github/workflows/qch.yml b/.github/workflows/qch.yml new file mode 100644 index 000000000..ef808cdc2 --- /dev/null +++ b/.github/workflows/qch.yml @@ -0,0 +1,204 @@ +# Builds the Qt Help (.qch) archive, and publishes it when a version is tagged. +# +# The raw copy of the website is not in this repository and is far too large to +# be. Rather than crawling en.cppreference.com on every run -- which the README +# asks us not to do -- the site copy is taken out of the source archive of an +# earlier release: `make release` packs `reference/` into +# `cppreference-doc-.tar.xz`, which is 13 MB and downloads in about a +# minute. Point `site_release` at the tag whose content you want, or refresh +# the content for a real release by running `make source` locally and publishing +# a new source archive. + +name: Qt Help archive + +on: + push: + branches: ['**'] + tags: ['v*'] + workflow_dispatch: + inputs: + site_release: + description: 'Release to take the reference/ site copy from (blank = latest)' + type: string + default: '' + site_repository: + description: 'Repository to take it from (blank = this one, then the ones it was forked from)' + type: string + default: '' + +# A run that has been superseded is of no interest; a tag builds on its own. +concurrency: + group: qch-${{ github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 60 + + permissions: + contents: write # the publishing step attaches the archives + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: | + requirements.txt + requirements-ci.txt + + - name: Install the Python dependencies + run: pip install -r requirements.txt -r requirements-ci.txt + + - name: Install qhelpgenerator + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qt6-documentation-tools + # Qt keeps it out of the way under /usr/lib/qt6/libexec, and the + # package is free to move it, so ask dpkg where it landed. + qhg=$(dpkg -L qt6-documentation-tools | grep -E '/qhelpgenerator$' | head -1) + test -x "$qhg" || { echo "::error::qhelpgenerator not found"; exit 1; } + echo "QHELPGENERATOR=$qhg" >> "$GITHUB_ENV" + + - name: Check the style + # pylint is left out on purpose: `config/pylintrc` predates pylint 3 and + # is rejected across the whole repository, so `test.sh` does not run + # clean here. Fixing that is its own change. + run: > + flake8 --config=config/flake8rc + *.py commands index_transform + gadgets/standard_revisions_tests/*.py tests + + - name: Run the tests + run: python -m unittest discover + + - name: Work out the version + run: | + version=$(sed -n 's/^VERSION=//p' Makefile) + test -n "$version" || { echo "::error::no VERSION in the Makefile"; exit 1; } + echo "VERSION=$version" >> "$GITHUB_ENV" + echo "Building version $version" + + # A tag that disagrees with the Makefile would publish archives whose + # names do not match the release they hang under. + if [ "$GITHUB_REF_TYPE" = tag ] && [ "$GITHUB_REF_NAME" != "v$version" ]; then + echo "::error::tag $GITHUB_REF_NAME does not match VERSION=$version in the Makefile" + exit 1 + fi + + - name: Fetch the copy of the website + env: + GH_TOKEN: ${{ github.token }} + SITE_TAG: ${{ inputs.site_release }} + SITE_REPO: ${{ inputs.site_repository }} + run: | + # Where to look, in order: what the run asked for, this repository, + # and the ones it was forked from. A fork has no releases of its + # own until it makes one, and the archive is the same either way. + candidates=() + if [ -n "$SITE_REPO" ]; then + candidates+=("$SITE_REPO") + fi + candidates+=("$GITHUB_REPOSITORY") + while read -r parent; do + if [ -n "$parent" ]; then + candidates+=("$parent") + fi + done < <(gh api "repos/$GITHUB_REPOSITORY" \ + -q '.parent.full_name // empty, .source.full_name // empty' \ + 2>/dev/null) + + for repo in "${candidates[@]}"; do + echo "Looking for the site copy in $repo" + # No tag asks gh for the latest release. + if gh release download ${SITE_TAG:+"$SITE_TAG"} \ + --repo "$repo" \ + --pattern 'cppreference-doc-*.tar.xz' \ + --clobber; then + site_repo="$repo" + break + fi + done + + if [ -z "${site_repo:-}" ]; then + echo "::error::no cppreference-doc-*.tar.xz release asset in ${candidates[*]}" + exit 1 + fi + echo "Site copy from $site_repo" + + tar xJf cppreference-doc-*.tar.xz + rm -rf reference + mv cppreference-doc-*/reference reference + rm -rf cppreference-doc-*/ cppreference-doc-*.tar.xz + + echo "Pages in the site copy: $(find reference -name '*.html' | wc -l)" + + - name: Build the archive + run: make doc_qch qhelpgenerator="$QHELPGENERATOR" + + - name: Check the archive + # The .qch is a SQLite database, so it can say for itself whether the + # pages, the images and both stylesheets arrived in it. + run: | + python - <<'EOF' + import sqlite3 + import sys + + db = sqlite3.connect('output/cppreference-doc-en-cpp.qch') + count, = db.execute('select count(*) from FileNameTable').fetchone() + sheets = sorted(name for (name,) in db.execute( + "select Name from FileNameTable where Name like 'common/offline%'")) + images, = db.execute( + "select count(*) from FileNameTable where Name like '%.svg'" + " or Name like '%.png'").fetchone() + + print('files: %d, images: %d' % (count, images)) + print('stylesheets: %s' % ', '.join(sheets)) + + if count < 6000: + sys.exit('only %d files in the archive' % count) + if len(sheets) != 3: + sys.exit('expected three stylesheets, found %d' % len(sheets)) + if images < 50: + sys.exit('only %d images in the archive' % images) + EOF + + # Handed to this, GitHub zips whatever it is given and serves the result + # under the artifact's name, so it gets the .qch itself rather than the + # release archives: those would come down as a zip holding an archive + # holding the one file anybody wanted, compressed twice over. + - uses: actions/upload-artifact@v4 + with: + name: cppreference-doc-en-cpp-${{ env.VERSION }} + path: output/cppreference-doc-en-cpp.qch + if-no-files-found: error + + - name: Package it as the release archives + # Release assets are served as they are, so these are the real thing: + # the same two names and the same format `make release` writes. + if: github.ref_type == 'tag' + run: | + mkdir -p release + pushd output > /dev/null + tar cJf "../release/qch-book-$VERSION.tar.xz" "cppreference-doc-en-cpp.qch" + zip -qr "../release/qch-book-$VERSION.zip" "cppreference-doc-en-cpp.qch" + popd > /dev/null + ls -lh release + + - name: Publish the release + if: github.ref_type == 'tag' + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then + # a release made by hand, or an earlier run of this workflow + gh release upload "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" \ + --clobber release/* + else + gh release create "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" \ + --title "$GITHUB_REF_NAME" --generate-notes \ + release/* + fi diff --git a/Makefile b/Makefile index 0a6b938c1..10e68bea1 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ DISTFILES= \ devhelp2qch.py \ export.py \ fix_devhelp-links.py \ + fixup_source.py \ index2autolinker.py \ index2browser.py \ index2ddg.py \ @@ -65,6 +66,8 @@ DISTFILES= \ preprocess.py \ preprocess-css.css \ preprocess_qch.py \ + qch-dark.css \ + qch-viewer.css \ test.sh \ xml_utils.py \ requirements.txt \ @@ -179,19 +182,19 @@ output/cppreference-doc-en-cpp.devhelp2: output/reference output/link-map.xml #build the .qch (QT help) file output/cppreference-doc-en-cpp.qch: output/qch-help-project-cpp.xml #qhelpgenerator only works if the project file is in the same directory as the documentation - cp "output/qch-help-project-cpp.xml" "output/reference_cssless/qch.qhp" + cp "output/qch-help-project-cpp.xml" "output/reference_qch/qch.qhp" - pushd "output/reference_cssless" > /dev/null; \ + pushd "output/reference_qch" > /dev/null; \ $(qhelpgenerator) "qch.qhp" -o "../cppreference-doc-en-cpp.qch"; \ popd > /dev/null - rm -f "output/reference_cssless/qch.qhp" + rm -f "output/reference_qch/qch.qhp" -output/qch-help-project-cpp.xml: output/cppreference-doc-en-cpp.devhelp2 output/reference_cssless +output/qch-help-project-cpp.xml: output/cppreference-doc-en-cpp.devhelp2 output/reference_qch #build the file list echo "" > "output/qch-files.xml" - pushd "output/reference_cssless" > /dev/null; \ + pushd "output/reference_qch" > /dev/null; \ find . -type f -not -iname "*.ttf" \ -exec echo ""'{}'"" \; | LC_ALL=C sort >> "../qch-files.xml" ; \ popd > /dev/null @@ -221,8 +224,8 @@ output/reference: mkdir -p output ./preprocess.py --src reference --dst output/reference -output/reference_cssless: output/reference - ./preprocess_qch.py --src output/reference --dst output/reference_cssless +output/reference_qch: output/reference + ./preprocess_qch.py --src output/reference --dst output/reference_qch # create indexes for the wiki indexes: @@ -241,8 +244,9 @@ source: mkdir "reference" pushd "reference" > /dev/null; \ - regex=".*index\\.php.*|.*action=.*|.*printable=.*|.*en.cppreference.com/book.*" \ - regex+="|.*/(User|Special|Talk|Help|File|Cppreference|WhatLinksHere|Template|Category|MediaWiki)(_talk)?(:|%253[aA]).*" ; \ + regex=".*index\\.php.*|.*rest\\.php.*|.*action=.*|.*printable=.*|.*en.cppreference.com/book.*" \ + regex+="|.*/(User|Special|Talk|Help|File|Cppreference|WhatLinksHere|Template|Category|MediaWiki)(_talk)?(:|%253[aA]).*" \ + regex+="|.*\\.cppreference\\.com/mwiki/.*" ; \ echo $$regex ; \ wget --adjust-extension --page-requisites --convert-links \ --force-directories --recursive --level=15 \ @@ -250,7 +254,16 @@ source: --reject-regex $$regex \ --timeout=5 --tries=50 --no-verbose \ --retry-connrefused --waitretry=10 --read-timeout=20 \ - https://en.cppreference.com/w/ ; \ + https://en.cppreference.com/ ; \ popd > /dev/null - ./export.py --url=https://en.cppreference.com/mwiki reference/cppreference-export-ns0,4,8,10.xml 0 4 8 10 + # the site still points at the pre-2026 locations for images and skin + # files; fetch those from where they live now + ./fixup_source.py --src reference + + # The wiki source of the pages is no longer part of the copy: since the + # 2026 upgrade the site answers 403 to any request that names more than + # one page -- Special:Export and the API alike, which is what export.py + # asks for -- and a copy worth having is not worth 19000 requests. + # Nothing in the build reads it; preprocess.py threw it away. + #./export.py --url=https://en.cppreference.com reference/cppreference-export-ns0,4,8,10.xml 0 4 8 10 diff --git a/README.md b/README.md index 0dd6dafa4..82087b8f9 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,13 @@ documentation formats are currently supported: `make install` installs the documentation into proper locations. 3) QT Help documentation format (.qch). Can be generated using `make doc_qch`. - `make install` installs the documentation into proper locations. + `make install` installs the documentation into proper locations. The pages + keep the site's own markup and stylesheet, which the help viewers that read + the archive -- Qt Assistant and Qt Creator, both of which render with + litehtml -- lay out much as a browser does. A dark stylesheet is shipped + beside the light one, and either viewer serves it when its own theme is + dark. 'commands/qch_css.py' builds both; 'qch-viewer.css' and 'qch-dark.css' + are the parts written by hand. Simply running `make all` will generate documentation in all three formats. @@ -43,7 +49,8 @@ Dependencies ------------ The package depends on 'wget' (>=1.15), 'python3', 'python3-lxml', -and 'qhelpgenerator' for the generation of the documentation. +'python3-cssutils' and 'qhelpgenerator' for the generation of the +documentation. See also -------- diff --git a/build_link_map.py b/build_link_map.py index 0ad37123d..147869b16 100755 --- a/build_link_map.py +++ b/build_link_map.py @@ -43,7 +43,10 @@ def build_link_map(directory): text = f.read() f.close() - m = re.search(r'', text) # noqa + # MediaWiki used to hand the page configuration to `mw.config.set` + # directly; since 1.43 it assigns it to `RLCONF` for the resource + # loader to pick up + m = re.search(r'', text) # noqa if not m: continue text = m.group(1) diff --git a/commands/preprocess.py b/commands/preprocess.py index 279b2e69c..fa54e5216 100644 --- a/commands/preprocess.py +++ b/commands/preprocess.py @@ -39,15 +39,66 @@ def move_dir_contents_to_dir(srcdir, dstdir): os.path.join(dstdir, fn)) +def is_data_file(fn): + # Everything the wiki serves next to the pages themselves: the skin, the + # style sheets and scripts the loader assembles, and the fonts and the + # favicon in case the site serves them again. + return (fn in ['skins', 'resources', 'favicon.ico'] or + fn.startswith('load.php') or fn.endswith('.ttf')) + + +def move_data_to_dir(src, dstdir): + # the data should be the same for all languages, so the first language to + # bring a file wins + dst = os.path.join(dstdir, os.path.basename(src)) + if os.path.isdir(src) and os.path.isdir(dst): + for fn in os.listdir(src): + move_data_to_dir(os.path.join(src, fn), dst) + os.rmdir(src) + elif not os.path.exists(dst): + shutil.move(src, dst) + + +def loader_modules(fn): + match = re.search(r'modules=([^&]*)', fn) + if match is None: + return set() + return set(urllib.parse.unquote(match.group(1)).split('|')) + + +def collapse_loader_files(data_path): + # Since the 2026 upgrade the loader serves a style sheet cut to measure + # for every page: the skin, plus whatever extensions that page happens to + # draw on. That is the same rules in a dozen combinations, and + # convert_loader_name() gives them all one name, so keep the one that + # asks for the most modules and throw the rest away. + groups = dict() + for fn in os.listdir(data_path): + if fn.startswith('load.php'): + groups.setdefault(convert_loader_name(fn), []).append(fn) + + for name, filenames in groups.items(): + filenames.sort(key=lambda fn: len(loader_modules(fn))) + keep = loader_modules(filenames[-1]) + + for fn in filenames[:-1]: + missing = loader_modules(fn) - keep + if missing: + msg = '{0} asks for {1}, which no other {2} has'.format( + fn, ', '.join(sorted(missing)), name) + raise Exception(msg) + os.remove(os.path.join(data_path, fn)) + + def rearrange_archive(root): # rearrange the archive. {root} here is output/reference # before - # {root}/en.cppreference.com/w/ : html - # {root}/en.cppreference.com/mwiki/ : data - # {root}/en.cppreference.com/ : data + # {root}/en.cppreference.com/ : html + # {root}/en.cppreference.com/load.php?... : data + # {root}/en.cppreference.com/{skins,resources}/ : data # ... (other languages) - # {root}/upload.cppreference.com/mwiki/ : data + # {root}/upload.cppreference.com/images/ : data # after # {root}/common/ : all common data @@ -56,33 +107,24 @@ def rearrange_archive(root): data_path = os.path.join(root, 'common') rmtree_if_exists(data_path) - shutil.move(os.path.join(root, 'upload.cppreference.com/mwiki'), data_path) - shutil.rmtree(os.path.join(root, 'upload.cppreference.com')) + os.makedirs(data_path) + + upload_path = os.path.join(root, 'upload.cppreference.com') + move_dir_contents_to_dir(upload_path, data_path) + shutil.rmtree(upload_path) for lang in ["en"]: - path = os.path.join(root, lang + ".cppreference.com/") - src_html_path = path + "w/" - src_data_path = path + "mwiki/" + path = os.path.join(root, lang + ".cppreference.com") html_path = os.path.join(root, lang) - if os.path.isdir(src_html_path): - shutil.move(src_html_path, html_path) + for fn in os.listdir(path): + if is_data_file(fn): + move_data_to_dir(os.path.join(path, fn), data_path) - if os.path.isdir(src_data_path): - # the skin files should be the same for all languages thus we - # can merge everything - move_dir_contents_to_dir(src_data_path, data_path) + # what is left are the pages + shutil.move(path, html_path) - # also copy the custom fonts - shutil.copy(os.path.join(path, 'DejaVuSansMonoCondensed60.ttf'), - data_path) - shutil.copy(os.path.join(path, 'DejaVuSansMonoCondensed75.ttf'), - data_path) - # and the favicon - shutil.copy(os.path.join(path, 'favicon.ico'), data_path) - - # remove what's left - shutil.rmtree(path) + collapse_loader_files(data_path) # remove the XML source file for fn in fnmatch.filter(os.listdir(root), 'cppreference-export*.xml'): @@ -94,13 +136,15 @@ def convert_loader_name(fn): # simplified name if "modules=site&only=scripts" in fn: return "site_scripts.js" - if "modules=site&only=styles" in fn: + if re.search(r'modules=site(\.styles)?&only=styles', fn): return "site_modules.css" if "modules=startup&only=scripts" in fn: return "startup_scripts.js" if re.search("modules=skins.*&only=scripts", fn): return "skin_scripts.js" - if re.search("modules=.*ext.*&only=styles", fn): + # the styles of the skin and of whatever extensions the page draws on, + # which the loader serves in one file + if re.search(r'modules=.*(ext|jquery|mediawiki|skins).*&only=styles', fn): return "ext.css" msg = 'Loader file {0} does not match any known files'.format(fn) raise Exception(msg) @@ -169,19 +213,41 @@ def find_html_files(root): def is_loader_link(target): - if re.match(r'https?://[a-z]+\.cppreference\.com/mwiki/load\.php', target): + # The loader sits at the root of the site, so the link is relative in a + # page that wget got the file for and absolute in one that it did not. + if re.search(r'(^|/)load\.php(?:[\?@]|%3[Ff])', target): return True return False def transform_loader_link(target, file, root): - # Absolute loader.php links need to be made relative + # Loader links point into the common data directory wherever they came + # from, so the link is built from scratch rather than adjusted abstarget = os.path.join(root, "common", convert_loader_name(target)) return os.path.relpath(abstarget, os.path.dirname(file)) +# The uploaded images moved out of /mwiki/ with the 2026 upgrade, but the +# pages still link to the old location. +UPLOAD_LINK = re.compile( + r'(?:https?:)?//upload\.cppreference\.com/(?:mwiki/)?images/') + + +def is_upload_link(target): + if UPLOAD_LINK.match(target): + return True + return False + + +def transform_upload_link(target, file, root): + path = urllib.parse.unquote(UPLOAD_LINK.sub('', target)) + abstarget = os.path.join(root, "common", "images", path) + return urllib.parse.quote( + os.path.relpath(abstarget, os.path.dirname(file))) + + def is_ranges_placeholder(target): - if re.match(r'https?://[a-z]+\.cppreference\.com/w/cpp/ranges(-[a-z]+)?-placeholder/.+', target): # noqa + if re.match(r'https?://[a-z]+\.cppreference\.com/(w/)?cpp/ranges(-[a-z]+)?-placeholder/.+', target): # noqa return True return False @@ -194,9 +260,9 @@ def transform_ranges_placeholder(target, file, root): repl = (r'\1/cpp/experimental/ranges/\2' if ranges else r'\1/cpp/\2') if 'ranges-placeholder' in target: - match = r'https?://([a-z]+)\.cppreference\.com/w/cpp/ranges-placeholder/(.+)' # noqa + match = r'https?://([a-z]+)\.cppreference\.com/(?:w/)?cpp/ranges-placeholder/(.+)' # noqa else: - match = r'https?://([a-z]+)\.cppreference\.com/w/cpp/ranges-([a-z]+)-placeholder/(.+)' # noqa + match = r'https?://([a-z]+)\.cppreference\.com/(?:w/)?cpp/ranges-([a-z]+)-placeholder/(.+)' # noqa repl += (r'/\3' if ranges else r'/ranges/\3') # Turn absolute placeholder link into site-relative link @@ -218,8 +284,6 @@ def trasform_relative_link(rename_map, target, file): assert params == '' path = urllib.parse.unquote(path) - path = path.replace('../../upload.cppreference.com/mwiki/', '../common/') - path = path.replace('../mwiki/', '../common/') dir, fn = os.path.split(path) new_fn = rename_map.get(fn) @@ -247,6 +311,9 @@ def transform_link(rename_map, target, file, root): if is_loader_link(target): return transform_loader_link(target, file, root) + if is_upload_link(target): + return transform_upload_link(target, file, root) + if is_ranges_placeholder(target): return transform_ranges_placeholder(target, file, root) @@ -256,6 +323,21 @@ def transform_link(rename_map, target, file, root): return trasform_relative_link(rename_map, target, file) +# Transforms the candidates of a srcset attribute: a comma separated list of +# links, each followed by the display density it is meant for. +def transform_srcset(rename_map, value, file, root): + candidates = [] + + for candidate in value.split(','): + parts = candidate.split() + if not parts: + continue + parts[0] = transform_link(rename_map, parts[0], file, root) + candidates.append(' '.join(parts)) + + return ', '.join(candidates) + + def has_class(el, *classes_to_check): value = el.get('class') if value is None: @@ -310,15 +392,19 @@ def remove_see_also(html): next.getparent().remove(next) -# remove Google Analytics scripts -def remove_google_analytics(html): - for el in html.xpath('/html/body/script'): +# remove analytics scripts; the New Relic agent is inlined into the head of +# every page and makes up the bulk of what a page weighs +def remove_analytics(html): + sources = ['google-analytics.com/ga.js', + 'static.cloudflareinsights.com'] + texts = ['google-analytics.com/ga.js', 'pageTracker', 'NREUM'] + + for el in html.xpath('/html/head/script') + html.xpath('/html/body/script'): if el.get('src') is not None: - if 'google-analytics.com/ga.js' in el.get('src'): + if any(src in el.get('src') for src in sources): el.getparent().remove(el) elif el.text is not None: - if 'google-analytics.com/ga.js' in el.text or \ - 'pageTracker' in el.text: + if any(text in el.text for text in texts): el.getparent().remove(el) @@ -340,12 +426,48 @@ def remove_ads(html): el.getparent().remove(el) +def unwrap_element(el): + # put what the element holds in its place, keeping the text around it + parent = el.getparent() + index = parent.index(el) + children = list(el) + + def add_text(text): + if not text: + return + if index == 0: + parent.text = (parent.text or '') + text + else: + parent[index - 1].tail = (parent[index - 1].tail or '') + text + + add_text(el.text) + + for offset, child in enumerate(children): + parent.insert(index + offset, child) + + tail = el.tail + parent.remove(el) + + if children: + children[-1].tail = (children[-1].tail or '') + (tail or '') + else: + add_text(tail) + + # remove links to file info pages (e.g. on images) def remove_fileinfo(html): - info = etree.XPath(r"//a[re:test(@href, 'https?://[a-z]+\.cppreference\.com/w/File:')]/..", # noqa + info = etree.XPath(r"//a[re:test(@href, 'https?://[a-z]+\.cppreference\.com/(w/)?File:')]", # noqa namespaces={'re':'http://exslt.org/regular-expressions'}) # noqa for el in info(html): - el.getparent().remove(el) + if has_class(el, 'mw-file-description'): + # Since the 2026 upgrade every figure is a link to the page that + # describes the file; the figure stays, the link goes. What the + # pages had before was a badge beside the figure, which is what + # the branch below takes out, wrapper and all. + unwrap_element(el) + else: + parent = el.getparent() + parent.getparent().remove(parent) # make custom footer @@ -359,7 +481,7 @@ def add_footer(html, root, fn): link = etree.SubElement(etree.SubElement(items, 'li'), 'a') url = re.sub(r'(..)/(.*)\.html', - r'https://\1.cppreference.com/w/\2', + r'https://\1.cppreference.com/\2', os.path.relpath(fn, root)) url = re.sub(r'(.*)/index', r'\1/', url) link.set('href', url) @@ -390,9 +512,17 @@ def preprocess_html_file(root, fn, rename_map): html = etree.parse(fn, parser) output = io.StringIO() + if html.getroot() is None: + # The parser hands back no element at all -- rather than raising -- + # for a document with nothing in it, which is what a page that came + # down truncated looks like. Leave it as it is, so that one bad file + # does not take the whole run with it. + print('Not an HTML page: {}'.format(fn), file=output) + return output.getvalue() + remove_unused_external(html) remove_noprint(html, keep_footer=True) - remove_google_analytics(html) + remove_analytics(html) remove_ads(html) remove_fileinfo(html) @@ -403,6 +533,9 @@ def preprocess_html_file(root, fn, rename_map): el.set('src', transform_link(rename_map, el.get('src'), fn, root)) for el in html.xpath('//*[@href]'): el.set('href', transform_link(rename_map, el.get('href'), fn, root)) + for el in html.xpath('//*[@srcset]'): + el.set('srcset', + transform_srcset(rename_map, el.get('srcset'), fn, root)) for err in list(parser.error_log): print("HTML WARN: {0}".format(err), file=output) @@ -411,27 +544,34 @@ def preprocess_html_file(root, fn, rename_map): return output.getvalue() -def preprocess_css_file(fn): - f = open(fn, "r", encoding='utf-8') - text = f.read() - f.close() - - # note that query string is not used in css files +def preprocess_css(text): + # the uploaded images and the skin files the site still points at the + # locations they had before the 2026 upgrade; both now live next to the + # style sheet in the common data directory + text = UPLOAD_LINK.sub('images/', text) + text = re.sub(r'(?:https?:)?//[a-z]+\.cppreference\.com/mwiki/skins/[^)\'"]*/images/', # noqa + 'skins/Cppreference2/resources/images/', text) - text = text.replace('../DejaVuSansMonoCondensed60.ttf', - 'DejaVuSansMonoCondensed60.ttf') - text = text.replace('../DejaVuSansMonoCondensed75.ttf', - 'DejaVuSansMonoCondensed75.ttf') + # the loader stamps a version onto the images it serves; the copy of the + # site has it as part of the file name, percent-encoded, and the files + # were renamed without it + text = re.sub(r'(url\([^)]*?)(?:[\?@]|%3[Ff])[^)\'"]*', r'\1', text) - text = text.replace('../../upload.cppreference.com/mwiki/images/', - 'images/') + # the fonts are no longer served; the rules that are left behind name + # locally installed fonts and the generic families to fall back to + text = re.sub(r'@font-face\s*{[^}]*(?:https?:)?//[a-z]+\.cppreference\.com/[^}]*}', # noqa + '', text) # QT Help viewer doesn't understand nth-child - text = text.replace('nth-child(1)', 'first-child') + return text.replace('nth-child(1)', 'first-child') - f = open(fn, "w", encoding='utf-8') - f.write(text) - f.close() + +def preprocess_css_file(fn): + with open(fn, "r", encoding='utf-8') as f: + text = f.read() + + with open(fn, "w", encoding='utf-8') as f: + f.write(preprocess_css(text)) def preprocess_startup_script(fn): diff --git a/commands/preprocess_cssless.py b/commands/preprocess_cssless.py deleted file mode 100644 index e6f9e0220..000000000 --- a/commands/preprocess_cssless.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright (C) 2018 Monika Kairaityte -# -# This file is part of cppreference-doc -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. - -import copy -import functools -import io -import logging -import os -import warnings - -import cssutils -from lxml import etree -from lxml.etree import strip_elements -from premailer import Premailer - - -def preprocess_html_merge_cssless(src_path, dst_path): - with open(src_path, 'r') as a_file: - content = a_file.read() - parser = etree.HTMLParser() - stripped = content.strip() - root = etree.fromstring(stripped, parser) - - output = preprocess_html_merge_css(root, src_path) - strip_style_tags(root) - remove_display_none(root) - convert_span_tables_to_tr_td(root) - convert_inline_block_elements_to_table(root) - convert_zero_td_width_to_nonzero(root) - convert_font_size_property_to_pt(root, 16) - convert_table_border_top_to_tr_background(root) - - head = os.path.dirname(dst_path) - os.makedirs(head, exist_ok=True) - - with open(dst_path, 'wb') as a_file: - root.getroottree().write(a_file, pretty_print=True, method="html", - encoding='utf-8') - return output - - -def silence_cssutils_warnings(): - log = logging.Logger('ignore') - output = io.StringIO() - handler = logging.StreamHandler(stream=output) - formatter = logging.Formatter('%(levelname)s, %(message)s') - handler.setFormatter(formatter) - log.addHandler(handler) - cssutils.log.setLog(log) - - return output - - -def preprocess_html_merge_css(root, src_path): - # cssutils_logging_handler of Premailer.__init__ is insufficient to silence - # warnings to stderr in non-verbose mode - output = silence_cssutils_warnings() - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - premailer = Premailer(root, base_url=src_path, - disable_link_rewrites=True, remove_classes=True) - root = premailer.transform().getroot() - - return output.getvalue() - - -def strip_style_tags(root): - strip_elements(root, 'style') - - -def needs_td_wrapper(element): - # element has table:row - if len(element.getchildren()) == 0: - return True - for el in element.getchildren(): - if has_css_property_value(el, 'display', 'table-row') or \ - has_css_property_value(el, 'display', 'table-cell'): - return False - return True - - -@functools.lru_cache(maxsize=None) -def cssutils_parse_style_cached_nocopy(style): - return cssutils.parseStyle(style) - - -def cssutils_parse_style_cached(style): - return copy.deepcopy(cssutils_parse_style_cached_nocopy(style)) - - -@functools.lru_cache(maxsize=None) -def get_css_style_property_value(style, prop_name): - atrib = cssutils_parse_style_cached_nocopy(style) - value = atrib.getPropertyCSSValue(prop_name) - if value: - return value.cssText - return None - - -@functools.lru_cache(maxsize=None) -def has_css_style_property_value(style, prop_name, prop_value): - value = get_css_style_property_value(style, prop_name) - if value and value == prop_value: - return True - return False - - -@functools.lru_cache(maxsize=None) -def remove_css_style_property(style, property_name): - atrib = cssutils_parse_style_cached(style) - atrib.removeProperty(property_name) - return atrib.getCssText(separator='') - - -def remove_css_property(element, property_name): - new_style = remove_css_style_property(element.get('style'), property_name) - if len(new_style) > 0: - element.set('style', new_style) - elif 'style' in element.attrib: - element.attrib.pop('style') - - -def get_css_property_value(el, prop_name): - return get_css_style_property_value(el.get('style'), prop_name) - - -def has_css_property_value(el, prop_name, prop_value): - return has_css_style_property_value(el.get('style'), prop_name, prop_value) - - -def set_css_property_value(el, prop_name, prop_value): - atrib = cssutils_parse_style_cached(el.get('style')) - atrib.setProperty(prop_name, prop_value) - el.set('style', atrib.getCssText(separator='')) - - -def convert_display_property_to_html_tag(element, element_tag, display_value): - str_attrib_value = element.get('style') - if str_attrib_value is None: - return False - if has_css_property_value(element, 'display', display_value): - element.tag = element_tag - remove_css_property(element, 'display') - return True - - -def convert_span_table_to_tr_td(table_el): - table_el.tag = 'table' - remove_css_property(table_el, 'display') - - for element in table_el.getchildren(): - tag_renamed = convert_display_property_to_html_tag( - element, 'tr', 'table-row') - if tag_renamed: - if needs_td_wrapper(element): - td = etree.Element('td') - for el in element.getchildren(): - element.remove(el) - td.append(el) - element.append(td) - td.text = element.text - element.text = None - else: - for child in element: - convert_display_property_to_html_tag( - child, 'td', 'table-cell') - - -def wrap_element(el, tag_name, style): - new_el = etree.Element(tag_name) - new_el.set('style', style) - el.addprevious(new_el) - new_el.insert(0, el) - - -def remove_display_none(root_el): - for el in root_el.xpath('//*[contains(@style, "display")]'): - if has_css_property_value(el, 'display', 'none'): - el.getparent().remove(el) - - -def convert_span_tables_to_tr_td(root_el): - # note that the following xpath expressions match only the prefix of the - # CSS property value - table_els = root_el.xpath('//span[contains(@style, "display:table")]') - - for table_el in table_els: - if has_css_property_value(table_el, 'display', 'table'): - convert_span_table_to_tr_td(table_el) - # wrap_element(table_el, 'div', 'display:inline') - # convert_span_table_to_tr_td(table_el) - - return root_el - - -def convert_inline_block_elements_to_table(root_el): - for el in root_el.xpath('//*[contains(@style, "display")]'): - if not has_css_property_value(el, 'display', 'inline-block') and \ - not has_css_property_value(el, 'display', 'inline-table'): - continue - - elements_to_put_into_table = [el] - el = el.getnext() - - # find subsequent inline block elements - while el is not None: - if has_css_property_value(el, 'display', 'inline-block') or \ - has_css_property_value(el, 'display', 'inline-table'): - elements_to_put_into_table.append(el) - else: - break - el = el.getnext() - - # only makes sense to put two or more to table - if len(elements_to_put_into_table) < 2: - continue - - # create table and put elements into it - table_el = etree.Element('table') - table_el.set('style', 'padding:0; margin:0; border:none;') - - elements_to_put_into_table[0].addprevious(table_el) - - tr = etree.SubElement(table_el, 'tr') - - for el in elements_to_put_into_table: - td = etree.SubElement(tr, 'td') - el.getparent().remove(el) - td.append(el) - - -def get_table_rows(table_el): - for el in table_el.iterchildren(['th', 'tr']): - yield el - for el in table_el.iterchildren(['thead', 'tbody', 'tfoot']): - for el2 in el.iterchildren(['th', 'tr']): - yield el2 - - -def get_max_number_of_columns(table_el): - max_tds = 0 - for row_el in get_table_rows(table_el): - max_tds = max(max_tds, len(list(row_el.iterchildren('td')))) - return max_tds - - -def clear_tr_border_top(tr_el): - for td_el in tr_el.iterchildren('td'): - remove_css_property(td_el, 'border-top') - - -def has_tr_border_top(tr_el): - for td_el in tr_el.iterchildren('td'): - if get_css_property_value(td_el, 'border-top') is not None: - return True - return False - - -def has_table_border_top(table_el): - for tr_el in get_table_rows(table_el): - if has_tr_border_top(tr_el): - return True - return False - - -def convert_table_border_top_to_tr_background(root_el): - for table_el in root_el.iter('table'): - if not has_table_border_top(table_el): - continue - - td_count = get_max_number_of_columns(table_el) - for tr_el in get_table_rows(table_el): - if has_tr_border_top(tr_el): - # TODO: handle border properties - clear_tr_border_top(tr_el) - border_tr = etree.Element('tr') - border_td = etree.SubElement(border_tr, 'td') - border_td.set('colspan', str(td_count)) - border_td.set('style', 'height:1px; font-size:1px; ' - 'background-color: #ccc;') - tr_el.addprevious(border_tr) - - -def convert_zero_td_width_to_nonzero(root_el): - for el in root_el.xpath('//*[contains(@style, "width")]'): - if has_css_property_value(el, 'width', '0%'): - el.attrib['width'] = "1px" - remove_css_property(el, 'width') - - for el in root_el.xpath('//*[contains(@width, "0%")]'): - el.attrib['width'] = "1px" - - -def apply_font_size(size, parent_size_pt): - size = size.strip() - - if size[-2:] == 'em': - value_number = float(size[:-2].strip()) - return value_number*parent_size_pt - - if size[-2:] in ['pt', 'px']: - return float(size[:-2].strip()) - - if size[-1] == '%': - value_number = float(size[:-1].strip())/100 - return value_number*parent_size_pt - - return parent_size_pt - - -def convert_font_size_property_to_pt_recurse(el, parent_size_pt): - size_value = get_css_property_value(el, "font-size") - - if size_value: - el_size_pt = apply_font_size(size_value, parent_size_pt) - set_css_property_value(el, "font-size", "{}pt".format(el_size_pt)) - else: - el_size_pt = parent_size_pt - - for child in el.getchildren(): - convert_font_size_property_to_pt_recurse(child, el_size_pt) - - -def convert_font_size_property_to_pt(root_el, default_size): - convert_font_size_property_to_pt_recurse(root_el, default_size) diff --git a/commands/preprocess_qch_html.py b/commands/preprocess_qch_html.py new file mode 100644 index 000000000..7361468ec --- /dev/null +++ b/commands/preprocess_qch_html.py @@ -0,0 +1,314 @@ +# Copyright (C) 2018 Monika Kairaityte +# Copyright (C) 2026 Cristian Adam +# +# This file is part of cppreference-doc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +# Fits a page of the website for the Qt Help (.qch) rendition. +# +# This pass used to flatten the site stylesheet into `style` attributes and +# rewrite the layout as nested tables, because the only help viewer then was +# Qt Assistant's, which is QTextBrowser and understands a small corner of +# CSS. Both viewers that read the archive now render with litehtml -- Qt +# Assistant since Qt 6.2, Qt Creator since 4.11 -- which follows a stylesheet +# link, so the pages keep their markup and their classes and are handed the +# site stylesheet whole; `commands/qch_css.py` builds it. +# +# What is left here is the handful of things a stylesheet cannot do. + +import functools +import os +import shutil + +from lxml import etree +from lxml.etree import strip_elements + +from commands import qch_css + + +def preprocess_html_file(src_path, dst_path, stylesheet_href): + with open(src_path, 'r', encoding='utf-8') as a_file: + root = etree.fromstring(a_file.read().strip(), etree.HTMLParser()) + + if root is None: + # Nothing to fit: the parser hands back no element at all -- rather + # than raising -- for a document with no content in it, which is what + # a page that came down truncated looks like. Carry it through as it + # is, so that one bad file does not take the whole run with it. + print('Not an HTML page: {}'.format(src_path)) + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + shutil.copyfile(src_path, dst_path) + return set(), (), set() + + page_css = lift_style_blocks(root) + strip_scripts(root) + drop_navbar_menus(root) + link_stylesheet(root, stylesheet_href) + clamp_rowspans(root) + unstack_vertical_headers(root) + rewrite_inline_styles(root) + # last, so that it sees the `style` attributes the passes above leave + color_classes = add_color_classes(root) + images = referenced_images(root, src_path) + + os.makedirs(os.path.dirname(dst_path), exist_ok=True) + with open(dst_path, 'wb') as a_file: + root.getroottree().write(a_file, pretty_print=True, method='html', + encoding='utf-8') + + return color_classes, page_css, images + + +def lift_style_blocks(root_el): + # The pages carry a - - - - - -
-
- - -

-std::basic_string

- - -
- - - - - -
-
-
 
- -
 
- -
 
-
std::basic_string
-
 
-
- - - - - - - - - - - - - - - - - - - - - - - - -
Defined in header <string> -
-
template<
-

    class CharT,
-    class Traits = std::char_traits<CharT>,
-    class Allocator = std::allocator<CharT>
-

-> class basic_string;
-
(1)
namespace pmr {
-

    template <class CharT, class Traits = std::char_traits<CharT>>
-    using basic_string = std::basic_string< CharT, Traits,
-                                            std::polymorphic_allocator<CharT>>
-

-}
-
(2) (since C++17)
-

The class template basic_string stores and manipulates sequences of char-like objects, which are non-array objects of trivial type. The class is dependent neither on the character type nor on the nature of operations on that type. The definitions of the operations are supplied via the Traits template parameter - a specialization of std::char_traits or a compatible traits class. Traits::char_type and CharT must name the same type; otherwise the behavior is undefined. -

- - - - - -
-

The elements of a basic_string are stored contiguously, that is, for a basic_string s, &*(s.begin() + n) == &*s.begin() + n for any n in [0, s.size()), or, equivalently, a pointer to s[0] can be passed to functions that expect a pointer to the first element of a CharT[] array. -

-
(since C++11)
-

std::basic_string satisfies the requirements of AllocatorAwareContainer, SequenceContainer and ContiguousContainer (since C++17) -

-

Several typedefs for common character types are provided: -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Defined in header <string>
-
Type - Definition -
std::string - std::basic_string<char> -
std::wstring - std::basic_string<wchar_t> -
std::u16string (C++11) - std::basic_string<char16_t> -
std::u32string (C++11) - std::basic_string<char32_t> -
std::pmr::string (C++17) - std::pmr::basic_string<char> -
std::pmr::wstring (C++17) - std::pmr::basic_string<wchar_t> -
std::pmr::u16string (C++17) - std::pmr::basic_string<char16_t> -
std::pmr::u32string (C++17) - std::pmr::basic_string<char32_t> -
-

Template parameters

- - - - - - - - - - - - - - - - - - -
CharT - - - character type -
Traits - - - traits class specifying the operations on the character type -
Allocator - - - Allocator type used to allocate internal storage -
-

Member types

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Member type - Definition -
traits_type - Traits -
value_type - CharT -
allocator_type - Allocator
size_type - - - - - - - - - -
Allocator::size_type(until C++11)
std::allocator_traits<Allocator>::size_type(since C++11)
difference_type - - - - - - - - - -
Allocator::difference_type(until C++11)
std::allocator_traits<Allocator>::difference_type(since C++11)
reference - - - - - - - - - -
Allocator::reference(until C++11)
value_type&(since C++11)
const_reference - - - - - - - - - -
Allocator::const_reference(until C++11)
const value_type&(since C++11)
pointer - - - - - - - - - -
Allocator::pointer(until C++11)
std::allocator_traits<Allocator>::pointer(since C++11)
const_pointer - - - - - - - - - -
Allocator::const_pointer(until C++11)
std::allocator_traits<Allocator>::const_pointer(since C++11)
iterator - RandomAccessIterator
const_iterator - Constant RandomAccessIterator
reverse_iterator - std::reverse_iterator<iterator>
const_reverse_iterator - std::reverse_iterator<const_iterator>
-

Member functions

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- constructs a basic_string
(public member function)
(destructor)
-
destroys the string, deallocating internal storage if used
(public member function) -
- assigns values to the string
(public member function)
- assign characters to a string
(public member function)
- returns the associated allocator
(public member function)
Element access -
-
- accesses the specified character with bounds checking
(public member function)
- accesses the specified character
(public member function)
- - -
(C++11)
-
accesses the first character
(public member function)
- - -
(C++11)
-
accesses the last character
(public member function)
- returns a pointer to the first character of a string
(public member function)
- returns a non-modifiable standard C character array version of the string
(public member function)
- returns a non-modifiable string_view into the entire string
(public member function)
Iterators -
-
- - -
- - -
(C++11)
-
returns an iterator to the beginning
(public member function)
- - -
- - -
(C++11)
-
returns an iterator to the end
(public member function)
- - -
- - -
(C++11)
-
returns a reverse iterator to the beginning
(public member function)
- - -
- - -
(C++11)
-
returns a reverse iterator to the end
(public member function)
Capacity -
-
- checks whether the string is empty
(public member function)
- returns the number of characters
(public member function)
- returns the maximum number of characters
(public member function)
- reserves storage
(public member function)
- returns the number of characters that can be held in currently allocated storage
(public member function)
- - -
(C++11)
-
reduces memory usage by freeing unused memory
(public member function)
Operations -
-
- clears the contents
(public member function)
- inserts characters
(public member function)
- removes characters
(public member function)
- appends a character to the end
(public member function)
- - -
(C++11)
-
removes the last character
(public member function)
- appends characters to the end
(public member function)
- appends characters to the end
(public member function)
- compares two strings
(public member function)
- - -
(C++20)
-
checks if the string starts with the given prefix
(public member function)
- - -
(C++20)
-
checks if the string ends with the given suffix
(public member function)
- replaces specified portion of a string
(public member function)
- returns a substring
(public member function)
- copies characters
(public member function)
- changes the number of characters stored
(public member function)
- swaps the contents
(public member function)
Search -
-
- find characters in the string
(public member function)
- find the last occurrence of a substring
(public member function)
- find first occurrence of characters
(public member function)
- find first absence of characters
(public member function)
- find last occurrence of characters
(public member function)
- find last absence of characters
(public member function)

Constants -

-
- - -
[static]
-
special value. The exact meaning depends on the context
(public static member constant)
-

Non-member functions

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- concatenates two strings or a string and a char
(function template)
- lexicographically compares two strings
(function template)
- specializes the std::swap algorithm
(function template)
Input/output -
-
- performs stream input and output on strings
(function template)
- read data from an I/O stream into a string
(function template)
Numeric conversions -
-
- - -
- - - -
(C++11)
(C++11)
(C++11)
-
converts a string to a signed integer
(function)
- - -
- - -
(C++11)
(C++11)
-
converts a string to an unsigned integer
(function)
- - -
- - - -
(C++11)
(C++11)
(C++11)
-
converts a string to a floating point value
(function)
- - -
(C++11)
-
converts an integral or floating point value to string
(function)
- - -
(C++11)
-
converts an integral or floating point value to wstring
(function)
-

Literals

- - - - - - - - - - - -
Defined in inline namespace std::literals::string_literals
-
- - -
(C++14)
-
Converts a character array literal to basic_string
(function)
-

Helper classes

- - - - - - - -
- hash support for strings
(class template specialization)
-

Deduction guides(since C++17)

- - - - -
- - - - - -
- - -
- -
-
- - - - - - - - - - - diff --git a/tests/preprocess_cssless_data/multiset_expected.html b/tests/preprocess_cssless_data/multiset_expected.html deleted file mode 100644 index b030a93b2..000000000 --- a/tests/preprocess_cssless_data/multiset_expected.html +++ /dev/null @@ -1,608 +0,0 @@ - - - -std::multiset - cppreference.com - - - - - - - - - - - - - -
-
- - -

-std::multiset

- - -
- - - - - -
-
-
 
- -
 
- -
 
-
std::multiset
-
 
-
- - - - - - - - - - - - - - - - - - - - - - - - -
Defined in header <set> -
-
template<
-

    class Key,
-    class Compare = std::less<Key>,
-    class Allocator = std::allocator<Key>
-

-> class multiset;
-
(1)
namespace pmr {
-

    template <class Key, class Compare = std::less<Key>>
-    using multiset = std::multiset<Key, Compare,
-                                   std::pmr::polymorphic_allocator<Key>>;
-

-}
-
(2) (since C++17)
-

std::multiset is an associative container that contains a sorted set of objects of type Key. Unlike set, multiple keys with equivalent values are allowed. Sorting is done using the key comparison function Compare. Search, insertion, and removal operations have logarithmic complexity. -

-

Everywhere the standard library uses the Compare concept, equivalence is determined by using the equivalence relation as described on Compare. In imprecise terms, two objects a and b are considered equivalent if neither compares less than the other: !comp(a, b) && !comp(b, a). -

-

The order of the elements that compare equivalent is the order of insertion and does not change. (since C++11) -

-

std::multiset meets the requirements of Container, AllocatorAwareContainer, AssociativeContainer and ReversibleContainer. -

-

Member types

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Member type - Definition -
key_type - Key
value_type - Key
size_type - Unsigned integer type (usually std::size_t)
difference_type - Signed integer type (usually std::ptrdiff_t)
key_compare - Compare
value_compare - Compare
allocator_type - Allocator
reference - - - - - - - - - -
Allocator::reference(until C++11)
value_type&(since C++11)
const_reference - - - - - - - - - -
Allocator::const_reference(until C++11)
const value_type&(since C++11)
pointer - - - - - - - - - -
Allocator::pointer(until C++11)
std::allocator_traits<Allocator>::pointer(since C++11)
const_pointer - - - - - - - - - -
Allocator::const_pointer(until C++11)
std::allocator_traits<Allocator>::const_pointer(since C++11)
iterator - - - - - - - - - -
BidirectionalIterator(until C++11)
Constant BidirectionalIterator -(since C++11)
const_iterator - Constant BidirectionalIterator
reverse_iterator - std::reverse_iterator<iterator>
const_reverse_iterator - std::reverse_iterator<const_iterator>
node_type(since C++17) - a specialization of node handle representing a container node
-

Member functions

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- constructs the multiset
(public member function)
- destructs the multiset
(public member function)
- assigns values to the container
(public member function)
- returns the associated allocator
(public member function)
Iterators -
-
- - -
- - -
-
returns an iterator to the beginning
(public member function)
- - -
- - -
-
returns an iterator to the end
(public member function)
- returns a reverse iterator to the beginning
(public member function)
- - -
- - -
-
returns a reverse iterator to the end
(public member function)
Capacity -
-
- checks whether the container is empty
(public member function)
- returns the number of elements
(public member function)
- returns the maximum possible number of elements
(public member function)
Modifiers -
-
- clears the contents
(public member function)
- inserts elements or nodes (since C++17)
(public member function)
- - -
(C++11)
-
constructs element in-place
(public member function)
- - -
(C++11)
-
constructs elements in-place using a hint
(public member function)
- erases elements
(public member function)
- swaps the contents
(public member function)
- - -
(C++17)
-
extracts nodes from the container
(public member function)
- - -
(C++17)
-
splices nodes from another container
(public member function)
Lookup -
-
- returns the number of elements matching specific key
(public member function)
- finds element with specific key
(public member function)
- returns range of elements matching a specific key
(public member function)
- returns an iterator to the first element not less than the given key
(public member function)
- returns an iterator to the first element greater than the given key
(public member function)
Observers -
-
- returns the function that compares keys
(public member function)
- returns the function that compares keys in objects of type value_type
(public member function)
-

Non-member functions

- - - - - - - - - - - - - -
- lexicographically compares the values in the multiset
(function template)
- specializes the std::swap algorithm
(function template)
-

Deduction guides(since C++17)

-

Notes

-

The member types iterator and const_iterator may be aliases to the same type. Since iterator is convertible to const_iterator, const_iterator should be used in function parameter lists to avoid violations of the One Definition Rule. -

- - - -
- - - - - -
- - -
- -
-
- - - - - - - - - - - diff --git a/tests/preprocess_cssless_data/basic_string.html b/tests/preprocess_qch_data/basic_string.html similarity index 100% rename from tests/preprocess_cssless_data/basic_string.html rename to tests/preprocess_qch_data/basic_string.html diff --git a/tests/preprocess_qch_data/basic_string_expected.html b/tests/preprocess_qch_data/basic_string_expected.html new file mode 100644 index 000000000..a4bfcf14f --- /dev/null +++ b/tests/preprocess_qch_data/basic_string_expected.html @@ -0,0 +1,854 @@ + + + +std::basic_string - cppreference.com + + + + + + + + + + +
+
+ + + +

+std::basic_string

+ + +
+ +
From cppreference.com
+ + +
< cpp‎ | string
+ + +
+
+
 
+ +
 
+ +
 
+
std::basic_string
+
 
+
+ + + + + + + + + + + + + + + + + + + + + +
Defined in header <string> +
+
template<
+

    class CharT,
+    class Traits = std::char_traits<CharT>,
+    class Allocator = std::allocator<CharT>
+

+> class basic_string;
+
(1)
namespace pmr {
+

    template <class CharT, class Traits = std::char_traits<CharT>>
+    using basic_string = std::basic_string< CharT, Traits,
+                                            std::polymorphic_allocator<CharT>>
+

+}
+
(2) (since C++17)
+

The class template basic_string stores and manipulates sequences of char-like objects, which are non-array objects of trivial type. The class is dependent neither on the character type nor on the nature of operations on that type. The definitions of the operations are supplied via the Traits template parameter - a specialization of std::char_traits or a compatible traits class. Traits::char_type and CharT must name the same type; otherwise the behavior is undefined. +

+ + + + + +
+

The elements of a basic_string are stored contiguously, that is, for a basic_string s, &*(s.begin() + n) == &*s.begin() + n for any n in [0, s.size()), or, equivalently, a pointer to s[0] can be passed to functions that expect a pointer to the first element of a CharT[] array. +

+
(since C++11)
+

std::basic_string satisfies the requirements of AllocatorAwareContainer, SequenceContainer and ContiguousContainer (since C++17) +

+

Several typedefs for common character types are provided: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Defined in header <string>
+
Type + Definition +
std::string + std::basic_string<char> +
std::wstring + std::basic_string<wchar_t> +
std::u16string (C++11) + std::basic_string<char16_t> +
std::u32string (C++11) + std::basic_string<char32_t> +
std::pmr::string (C++17) + std::pmr::basic_string<char> +
std::pmr::wstring (C++17) + std::pmr::basic_string<wchar_t> +
std::pmr::u16string (C++17) + std::pmr::basic_string<char16_t> +
std::pmr::u32string (C++17) + std::pmr::basic_string<char32_t> +
+

Template parameters

+ + + + + + + + + + + + + + + + + + +
CharT + - + character type +
Traits + - + traits class specifying the operations on the character type +
Allocator + - + Allocator type used to allocate internal storage +
+

Member types

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Member type + Definition +
traits_type + Traits +
value_type + CharT +
allocator_type + Allocator
size_type + + + + + + + + + +
Allocator::size_type(until C++11)
std::allocator_traits<Allocator>::size_type(since C++11)
difference_type + + + + + + + + + +
Allocator::difference_type(until C++11)
std::allocator_traits<Allocator>::difference_type(since C++11)
reference + + + + + + + + + +
Allocator::reference(until C++11)
value_type&(since C++11)
const_reference + + + + + + + + + +
Allocator::const_reference(until C++11)
const value_type&(since C++11)
pointer + + + + + + + + + +
Allocator::pointer(until C++11)
std::allocator_traits<Allocator>::pointer(since C++11)
const_pointer + + + + + + + + + +
Allocator::const_pointer(until C++11)
std::allocator_traits<Allocator>::const_pointer(since C++11)
iterator + RandomAccessIterator
const_iterator + Constant RandomAccessIterator
reverse_iterator + std::reverse_iterator<iterator>
const_reverse_iterator + std::reverse_iterator<const_iterator>
+

Member functions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ constructs a basic_string
(public member function)
(destructor)
+
destroys the string, deallocating internal storage if used
(public member function) +
+ assigns values to the string
(public member function)
+ assign characters to a string
(public member function)
+ returns the associated allocator
(public member function)
Element access +
+
+ accesses the specified character with bounds checking
(public member function)
+ accesses the specified character
(public member function)
+ +
(C++11)
+
+
accesses the first character
(public member function)
+ +
(C++11)
+
+
accesses the last character
(public member function)
+ returns a pointer to the first character of a string
(public member function)
+ returns a non-modifiable standard C character array version of the string
(public member function)
+ returns a non-modifiable string_view into the entire string
(public member function)
Iterators +
+
+ +
(C++11)
+
+
returns an iterator to the beginning
(public member function)
+ +
(C++11)
+
+
returns an iterator to the end
(public member function)
+ +
(C++11)
+
+
returns a reverse iterator to the beginning
(public member function)
+ +
(C++11)
+
+
returns a reverse iterator to the end
(public member function)
Capacity +
+
+ checks whether the string is empty
(public member function)
+ returns the number of characters
(public member function)
+ returns the maximum number of characters
(public member function)
+ reserves storage
(public member function)
+ returns the number of characters that can be held in currently allocated storage
(public member function)
+ +
(C++11)
+
+
reduces memory usage by freeing unused memory
(public member function)
Operations +
+
+ clears the contents
(public member function)
+ inserts characters
(public member function)
+ removes characters
(public member function)
+ appends a character to the end
(public member function)
+ +
(C++11)
+
+
removes the last character
(public member function)
+ appends characters to the end
(public member function)
+ appends characters to the end
(public member function)
+ compares two strings
(public member function)
+ +
(C++20)
+
+
checks if the string starts with the given prefix
(public member function)
+ +
(C++20)
+
+
checks if the string ends with the given suffix
(public member function)
+ replaces specified portion of a string
(public member function)
+ returns a substring
(public member function)
+ copies characters
(public member function)
+ changes the number of characters stored
(public member function)
+ swaps the contents
(public member function)
Search +
+
+ find characters in the string
(public member function)
+ find the last occurrence of a substring
(public member function)
+ find first occurrence of characters
(public member function)
+ find first absence of characters
(public member function)
+ find last occurrence of characters
(public member function)
+ find last absence of characters
(public member function)

Constants +

+
+ +
[static]
+
+
special value. The exact meaning depends on the context
(public static member constant)
+

Non-member functions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ concatenates two strings or a string and a char
(function template)
+ lexicographically compares two strings
(function template)
+ specializes the std::swap algorithm
(function template)
Input/output +
+
+ performs stream input and output on strings
(function template)
+ read data from an I/O stream into a string
(function template)
Numeric conversions +
+
+ +
(C++11)(C++11)(C++11)
+
+
converts a string to a signed integer
(function)
+ +
(C++11)(C++11)
+
+
converts a string to an unsigned integer
(function)
+ +
(C++11)(C++11)(C++11)
+
+
converts a string to a floating point value
(function)
+ +
(C++11)
+
+
converts an integral or floating point value to string
(function)
+ +
(C++11)
+
+
converts an integral or floating point value to wstring
(function)
+

Literals

+ + + + + + + + + + +
Defined in inline namespace std::literals::string_literals
+
+ +
(C++14)
+
+
Converts a character array literal to basic_string
(function)
+

Helper classes

+ + + + + + +
+ hash support for strings
(class template specialization)
+

Deduction guides(since C++17)

+ + + + +
+ + + + + +
+ + +
+ +
+
+ + + + + + + + + + + diff --git a/tests/preprocess_cssless_data/ext.css b/tests/preprocess_qch_data/ext.css similarity index 100% rename from tests/preprocess_cssless_data/ext.css rename to tests/preprocess_qch_data/ext.css diff --git a/tests/preprocess_cssless_data/multiset.html b/tests/preprocess_qch_data/multiset.html similarity index 100% rename from tests/preprocess_cssless_data/multiset.html rename to tests/preprocess_qch_data/multiset.html diff --git a/tests/preprocess_qch_data/multiset_expected.html b/tests/preprocess_qch_data/multiset_expected.html new file mode 100644 index 000000000..c64e70de3 --- /dev/null +++ b/tests/preprocess_qch_data/multiset_expected.html @@ -0,0 +1,512 @@ + + + +std::multiset - cppreference.com + + + + + + + + + + +
+
+ + + +

+std::multiset

+ + +
+ +
From cppreference.com
+ + +
< cpp‎ | container
+ + +
+
+
 
+ +
 
+ +
 
+
std::multiset
+
 
+
+ + + + + + + + + + + + + + + + + + + + + +
Defined in header <set> +
+
template<
+

    class Key,
+    class Compare = std::less<Key>,
+    class Allocator = std::allocator<Key>
+

+> class multiset;
+
(1)
namespace pmr {
+

    template <class Key, class Compare = std::less<Key>>
+    using multiset = std::multiset<Key, Compare,
+                                   std::pmr::polymorphic_allocator<Key>>;
+

+}
+
(2) (since C++17)
+

std::multiset is an associative container that contains a sorted set of objects of type Key. Unlike set, multiple keys with equivalent values are allowed. Sorting is done using the key comparison function Compare. Search, insertion, and removal operations have logarithmic complexity. +

+

Everywhere the standard library uses the Compare concept, equivalence is determined by using the equivalence relation as described on Compare. In imprecise terms, two objects a and b are considered equivalent if neither compares less than the other: !comp(a, b) && !comp(b, a). +

+

The order of the elements that compare equivalent is the order of insertion and does not change. (since C++11) +

+

std::multiset meets the requirements of Container, AllocatorAwareContainer, AssociativeContainer and ReversibleContainer. +

+

Member types

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Member type + Definition +
key_type + Key
value_type + Key
size_type + Unsigned integer type (usually std::size_t)
difference_type + Signed integer type (usually std::ptrdiff_t)
key_compare + Compare
value_compare + Compare
allocator_type + Allocator
reference + + + + + + + + + +
Allocator::reference(until C++11)
value_type&(since C++11)
const_reference + + + + + + + + + +
Allocator::const_reference(until C++11)
const value_type&(since C++11)
pointer + + + + + + + + + +
Allocator::pointer(until C++11)
std::allocator_traits<Allocator>::pointer(since C++11)
const_pointer + + + + + + + + + +
Allocator::const_pointer(until C++11)
std::allocator_traits<Allocator>::const_pointer(since C++11)
iterator + + + + + + + + + +
BidirectionalIterator(until C++11)
Constant BidirectionalIterator +(since C++11)
const_iterator + Constant BidirectionalIterator
reverse_iterator + std::reverse_iterator<iterator>
const_reverse_iterator + std::reverse_iterator<const_iterator>
node_type(since C++17) + a specialization of node handle representing a container node
+

Member functions

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ constructs the multiset
(public member function)
+ destructs the multiset
(public member function)
+ assigns values to the container
(public member function)
+ returns the associated allocator
(public member function)
Iterators +
+
+ returns an iterator to the beginning
(public member function)
+ +
+
+
returns an iterator to the end
(public member function)
+ returns a reverse iterator to the beginning
(public member function)
+ returns a reverse iterator to the end
(public member function)
Capacity +
+
+ checks whether the container is empty
(public member function)
+ returns the number of elements
(public member function)
+ returns the maximum possible number of elements
(public member function)
Modifiers +
+
+ clears the contents
(public member function)
+ inserts elements or nodes (since C++17)
(public member function)
+ +
(C++11)
+
+
constructs element in-place
(public member function)
+ +
(C++11)
+
+
constructs elements in-place using a hint
(public member function)
+ erases elements
(public member function)
+ swaps the contents
(public member function)
+ +
(C++17)
+
+
extracts nodes from the container
(public member function)
+ +
(C++17)
+
+
splices nodes from another container
(public member function)
Lookup +
+
+ returns the number of elements matching specific key
(public member function)
+ finds element with specific key
(public member function)
+ returns range of elements matching a specific key
(public member function)
+ returns an iterator to the first element not less than the given key
(public member function)
+ returns an iterator to the first element greater than the given key
(public member function)
Observers +
+
+ returns the function that compares keys
(public member function)
+ returns the function that compares keys in objects of type value_type
(public member function)
+

Non-member functions

+ + + + + + + + + + + +
+ lexicographically compares the values in the multiset
(function template)
+ specializes the std::swap algorithm
(function template)
+

Deduction guides(since C++17)

+

Notes

+

The member types iterator and const_iterator may be aliases to the same type. Since iterator is convertible to const_iterator, const_iterator should be used in function parameter lists to avoid violations of the One Definition Rule. +

+ + + +
+ + + + + +
+ + +
+ +
+
+ + + + + + + + + + + diff --git a/tests/preprocess_cssless_data/site_modules.css b/tests/preprocess_qch_data/site_modules.css similarity index 100% rename from tests/preprocess_cssless_data/site_modules.css rename to tests/preprocess_qch_data/site_modules.css diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py index 3c0c95259..7c3e5d9a7 100644 --- a/tests/test_preprocess.py +++ b/tests/test_preprocess.py @@ -31,15 +31,19 @@ from commands.preprocess import convert_loader_name from commands.preprocess import has_class from commands.preprocess import is_external_link +from commands.preprocess import is_loader_link from commands.preprocess import is_ranges_placeholder +from commands.preprocess import is_upload_link +from commands.preprocess import preprocess_css from commands.preprocess import remove_ads +from commands.preprocess import remove_analytics from commands.preprocess import remove_fileinfo -from commands.preprocess import remove_google_analytics from commands.preprocess import remove_noprint from commands.preprocess import remove_see_also from commands.preprocess import remove_unused_external from commands.preprocess import rename_files from commands.preprocess import transform_ranges_placeholder +from commands.preprocess import transform_srcset from commands.preprocess import trasform_relative_link @@ -77,6 +81,156 @@ def test_convert_loader_name(self): with self.assertRaises(Exception): convert_loader_name('') + def test_convert_loader_name_after_upgrade(self): + # the 2026 upgrade moved the loader to the root of the site and split + # the site styles off into a module of their own + url = '/load.php?lang=en&modules=startup&only=scripts&raw=1&skin=cppreference2' # noqa + self.assertEqual('startup_scripts.js', convert_loader_name(url)) + + url = '/load.php?lang=en&modules=site.styles&only=styles&skin=cppreference2' # noqa + self.assertEqual('site_modules.css', convert_loader_name(url)) + + url = '/load.php?lang=en&modules=ext.pygments%7Cskins.cppreference2.styles&only=styles&skin=cppreference2' # noqa + self.assertEqual('ext.css', convert_loader_name(url)) + + +class TestIsLoaderLink(unittest.TestCase): + def test_is_loader_link(self): + loader = [ + 'http://en.cppreference.com/mwiki/load.php?modules=site&only=styles', # noqa + 'https://en.cppreference.com/load.php?modules=site.styles&only=styles', # noqa + '/load.php?modules=site.styles&only=styles', + '../../load.php?modules=site.styles&only=styles', + # wget writes the query as part of the file name + '../load.php@modules=site.styles&only=styles', + ] + for link in loader: + self.assertTrue(is_loader_link(link), + msg="Should be a loader link: {}".format(link)) + + other = [ + 'https://en.cppreference.com/cpp/io.html', + '../io/basic_ostream.html', + 'upload.php?foo=bar', + ] + for link in other: + self.assertFalse(is_loader_link(link), + msg="Should not be a loader link: {}".format(link)) + + +class TestIsUploadLink(unittest.TestCase): + def test_is_upload_link(self): + upload = [ + 'https://upload.cppreference.com/mwiki/images/0/06/foo.svg', + 'http://upload.cppreference.com/mwiki/images/0/06/foo.svg', + '//upload.cppreference.com/images/0/06/foo.svg', + ] + for link in upload: + self.assertTrue(is_upload_link(link), + msg="Should be an upload link: {}".format(link)) + + other = [ + 'https://en.cppreference.com/images/0/06/foo.svg', + '../common/images/0/06/foo.svg', + ] + for link in other: + self.assertFalse(is_upload_link(link), + msg="Should not be an upload link: {}".format(link)) # noqa + + +class TestRemoveFileinfo(unittest.TestCase): + def parse(self, body): + return etree.fromstring(body, etree.HTMLParser()).getroottree() + + def test_figure_keeps_its_image(self): + # since the 2026 upgrade every figure is a link to the page that + # describes the file + html = self.parse( + '

before' + '' + '' + '' + 'after

') + + remove_fileinfo(html) + + self.assertEqual( + ['../common/images/1/1b/range-begin-end.svg'], + html.xpath('//img/@src')) + self.assertEqual([], html.xpath('//a')) + self.assertEqual('beforeafter', ''.join(html.xpath('//p//text()'))) + + def test_badge_beside_the_figure_goes(self): + # what the pages had before: a badge of its own beside the figure + html = self.parse( + '
' + '
' + '' + 'About this image' + '
') + + remove_fileinfo(html) + + self.assertEqual(['../common/images/0/06/diagram.svg'], + html.xpath('//img/@src')) + + +class TestTransformSrcset(unittest.TestCase): + def test_transform_srcset(self): + # the links of a srcset are the links of the page, each followed by + # the display density it is meant for + value = ('https://upload.cppreference.com/mwiki/images/thumb/9/91/math-atan2.png/428px-math-atan2.png 1.5x, ' # noqa + 'https://upload.cppreference.com/mwiki/images/thumb/9/91/math-atan2.png/570px-math-atan2.png 2x') # noqa + + self.assertEqual( + '../../../../common/images/thumb/9/91/math-atan2.png/428px-math-atan2.png 1.5x, ' # noqa + '../../../../common/images/thumb/9/91/math-atan2.png/570px-math-atan2.png 2x', # noqa + transform_srcset({}, value, + 'output/reference/en/cpp/numeric/math/atan2.html', + 'output/reference')) + + +class TestPreprocessCss(unittest.TestCase): + def test_images_of_the_site(self): + # the site points at the locations these had before the 2026 upgrade + text = ('a{background-image:url(https://upload.cppreference.com/mwiki/images/2/23/Icons-mini-file_acrobat.gif)}\n' # noqa + 'b{background-image:url(//en.cppreference.com/mwiki/skins/cppreference2/images/navbar-tab.png)}\n' # noqa + 'c{background-image:url(https://en.cppreference.com/mwiki/skins/vector/images/external-link-ltr-icon.png)}') # noqa + + self.assertEqual( + 'a{background-image:url(images/2/23/Icons-mini-file_acrobat.gif)}\n' # noqa + 'b{background-image:url(skins/Cppreference2/resources/images/navbar-tab.png)}\n' # noqa + 'c{background-image:url(skins/Cppreference2/resources/images/external-link-ltr-icon.png)}', # noqa + preprocess_css(text)) + + def test_version_of_the_loader(self): + # the version is part of the file name in the copy of the site, and + # the files were renamed without it + text = ('a{background-image:url(skins/Cppreference2/resources/images/tab-break.png%3Ff9125)}\n' # noqa + 'b{background-image:url("page-fade.png?577de")}') + + self.assertEqual( + 'a{background-image:url(skins/Cppreference2/resources/images/tab-break.png)}\n' # noqa + 'b{background-image:url("page-fade.png")}', + preprocess_css(text)) + + def test_fonts_of_the_site(self): + # the site no longer serves the fonts; what is left of the rule names + # the face as it is installed locally, and the stacks that ask for it + # fall back on a generic family + text = ('@font-face{font-family:"DejaVuSans" ;src:local("DejaVu Sans"),url(//en.cppreference.com/DejaVuSans.ttf) format("truetype");font-weight:normal}\n' # noqa + 'body{font-family:DejaVuSans,"DejaVu Sans",arial,sans-serif}') + + self.assertEqual( + '\nbody{font-family:DejaVuSans,"DejaVu Sans",arial,sans-serif}', + preprocess_css(text)) + + def test_nth_child(self): + self.assertEqual('tr > td:first-child{color:red}', + preprocess_css('tr > td:nth-child(1){color:red}')) + class TestHasClass(unittest.TestCase): def test_has_class(self): @@ -161,6 +315,9 @@ def test_is_ranges_placeholder(self): 'http://en.cppreference.com/w/cpp/ranges-algorithm-placeholder/all_any_none_of', # noqa 'http://en.cppreference.com/w/cpp/ranges-iterator-placeholder/dangling', # noqa 'http://en.cppreference.com/w/cpp/ranges-utility-placeholder/swap', # noqa + # the 2026 upgrade moved the wiki out of /w/ + 'http://en.cppreference.com/cpp/ranges-placeholder/concepts/Assignable', # noqa + 'http://en.cppreference.com/cpp/ranges-algorithm-placeholder/all_any_none_of', # noqa ] for target in match: self.assertTrue(is_ranges_placeholder(target), @@ -280,8 +437,8 @@ def test_remove_see_also(self): remove_see_also(self.html) self.check_output("fabs_seealso.html") - def test_remove_google_analytics(self): - remove_google_analytics(self.html) + def test_remove_analytics(self): + remove_analytics(self.html) self.check_output("fabs_ga.html") def test_remove_ads(self): @@ -426,14 +583,6 @@ def test_transform_relative_link(self): '../f1?some=param#p2', '../f1#p2'), - ('output/dir1/sub2/Conflict.html', - '../mwiki/site.css', - '../common/site.css'), - - ('output/dir1/sub2/Conflict.html', - '../../upload.cppreference.com/mwiki/site.css', - '../common/site.css'), - ('output/dir1/f2', 'sub1/invalid*.txt', 'sub1/invalid_star_.txt'), diff --git a/tests/test_preprocess_cssless.py b/tests/test_preprocess_cssless.py deleted file mode 100644 index 64864ccf2..000000000 --- a/tests/test_preprocess_cssless.py +++ /dev/null @@ -1,657 +0,0 @@ -# Copyright (C) 2018 Monika Kairaityte -# -# This file is part of cppreference-doc -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. - -import os -import unittest - -from lxml import etree - -from commands.preprocess_cssless import apply_font_size -from commands.preprocess_cssless import convert_font_size_property_to_pt -from commands.preprocess_cssless import convert_inline_block_elements_to_table -from commands.preprocess_cssless import convert_span_tables_to_tr_td -from commands.preprocess_cssless import \ - convert_table_border_top_to_tr_background -from commands.preprocess_cssless import convert_zero_td_width_to_nonzero -from commands.preprocess_cssless import preprocess_html_merge_cssless -from commands.preprocess_cssless import silence_cssutils_warnings - - -class TestPreprocessHtmlMergeCss(unittest.TestCase): - def test_preprocess_html_merge_cssless(self): - dir_path = os.path.dirname(__file__) - src_path = os.path.join( - dir_path, 'preprocess_cssless_data/multiset.html') - dst_path = os.path.join( - dir_path, 'preprocess_cssless_data/multiset_out.html') - expected_path = os.path.join( - dir_path, 'preprocess_cssless_data/multiset_expected.html') - - preprocess_html_merge_cssless(src_path, dst_path) - - with open(dst_path, 'r') as a_file: - test = a_file.read() - - with open(expected_path, 'r') as a_file: - expected = a_file.read() - - self.assertEqual(test, expected) - os.remove(dst_path) - - def test_preprocess_html_merge_cssless2(self): - dir_path = os.path.dirname(__file__) - src_path = os.path.join( - dir_path, 'preprocess_cssless_data/basic_string.html') - dst_path = os.path.join( - dir_path, 'preprocess_cssless_data/basic_string_out.html') - expected_path = os.path.join( - dir_path, 'preprocess_cssless_data/basic_string_expected.html') - - preprocess_html_merge_cssless(src_path, dst_path) - - with open(dst_path, 'r') as a_file: - test = a_file.read() - - with open(expected_path, 'r') as a_file: - expected = a_file.read() - - self.assertEqual(test, expected) - os.remove(dst_path) - - -class HTMLTestBase(unittest.TestCase): - def setUp(self): - self.maxDiff = None - silence_cssutils_warnings() - - def assert_converts_html(self, input, expected_output, function): - input = '{0}'.format(input) - expected_output = \ - '{0}'.format(expected_output) - - parser = etree.HTMLParser() - root = etree.fromstring(input, parser) - - root = function(root) - - output = etree.tostring(root, encoding=str, method='xml') - - self.assertEqual(expected_output, output) - - -class TestConvertSpanTablesToTrTd(HTMLTestBase): - def assert_processes_table(self, input, expected_output): - def test_fun(root): - convert_span_tables_to_tr_td(root) - return root - - self.assert_converts_html(input, expected_output, test_fun) - - def test_normal_table(self): - input = '''\ -
- - - - 1 - 2 - - - -
-''' # noqa - - expected = '''\ -
- - - - - - -
12
-
-
-''' - self.assert_processes_table(input, expected) - - def test_wraps_table_row_text(self): - input = '''\ -
- - - little text - - -
-''' - expected = '''\ -
- - - -
little text
-
-
-''' - self.assert_processes_table(input, expected) - - def test_wraps_into_td_table_row_children(self): - input = '''\ -
- - - - blabla - blabla2 - - - -
-''' - expected = '''\ -
- - - -
- blabla - blabla2 -
-
-
-''' - self.assert_processes_table(input, expected) - - def test_wraps_into_td_table_row_children_with_style(self): - input = '''\ -
- - - - (C++11) - - - -
-''' - expected = '''\ -
- - - -
- (C++11) -
-
-
-''' - self.assert_processes_table(input, expected) - - def test_wraps_into_td_table_row_children_with_tags(self): - input = '''\ -
- - - - -
bla
-
- blabla2 -
-
-
-
-''' - expected = '''\ -
- - - -
- -
bla
-
- blabla2 -
-
-
-''' - self.assert_processes_table(input, expected) - - def test_does_not_wrap_into_td_children_when_not_in_table_row(self): - input = '''\ -
- - - - 2 - - - -
-''' - expected = '''\ -
- - - - 2 - -
-
-
-''' - self.assert_processes_table(input, expected) - - def test_does_not_wrap_children_when_sibling_is_td(self): - input = '''\ -
- - - - 1 - 2 - - - -
-''' - expected = '''\ -
- - - - 1 - - -
2
-
-
-''' - self.assert_processes_table(input, expected) - - def test_does_not_convert_tr_when_parent_is_not_table(self): - input = '''\ -
- - - - - blabla - - - - -
-''' - expected = '''\ -
- - - - - blabla - - -
-
-
-''' - self.assert_processes_table(input, expected) - - def test_does_not_convert_td_when_parent_is_not_tr(self): - input = '''\ -
- - - - - blabla - - - - -
-''' - expected = '''\ -
- - - - - blabla - - -
-
-
-''' - self.assert_processes_table(input, expected) - - -class TestConvertInlineBlockElementsToTable(HTMLTestBase): - def perform_test(self, input, expected_output): - def test_fun(root): - convert_inline_block_elements_to_table(root) - return root - - self.assert_converts_html(input, expected_output, test_fun) - - def test_does_not_convert_single(self): - input = '''\ -
-
-
-''' - self.perform_test(input, input) - - def test_converts_multiple(self): - input = '''\ -
-
-
-
-''' - expected = '''\ -
-
-
-
-''' # noqa - self.perform_test(input, expected) - - -class TestConvertZeroTdWidthToNonzero(HTMLTestBase): - def test_css_property(self): - def test_fun(root): - convert_zero_td_width_to_nonzero(root) - return root - - input = '''\ - - - - -''' - expected = '''\ - - - - -''' - self.assert_converts_html(input, expected, test_fun) - - def test_attribute(self): - def test_fun(root): - convert_zero_td_width_to_nonzero(root) - return root - - input = '''\ - - - - -''' - expected = '''\ - - - - -''' - self.assert_converts_html(input, expected, test_fun) - - -class TestApplyFontSize(unittest.TestCase): - def test_em(self): - self.assertEqual(10, apply_font_size('1em', 10)) - self.assertEqual(8, apply_font_size('0.8em', 10)) - self.assertEqual(10, apply_font_size('1.0em', 10)) - self.assertEqual(15, apply_font_size('1.5em', 10)) - - self.assertEqual(15, apply_font_size('1.5 em', 10)) - self.assertEqual(15, apply_font_size(' 1.5em ', 10)) - self.assertEqual(15, apply_font_size('1.5em ', 10)) - - def test_px(self): - self.assertEqual(8, apply_font_size('8px', 10)) - self.assertEqual(10, apply_font_size('10px', 10)) - self.assertEqual(15, apply_font_size('15px', 10)) - - self.assertEqual(15, apply_font_size('15 px', 10)) - self.assertEqual(15, apply_font_size(' 15px ', 10)) - self.assertEqual(15, apply_font_size('15px ', 10)) - - def test_ptc(self): - self.assertEqual(8, apply_font_size('80%', 10)) - self.assertEqual(10, apply_font_size('100%', 10)) - self.assertEqual(15, apply_font_size('150%', 10)) - - self.assertEqual(15, apply_font_size('150 %', 10)) - self.assertEqual(15, apply_font_size(' 150% ', 10)) - self.assertEqual(15, apply_font_size('150% ', 10)) - - -class TestConvertFontSizePropertyToPt(HTMLTestBase): - def test_simple(self): - def test_fun(root): - convert_font_size_property_to_pt(root, 10) - return root - - input = '''\ -
-text -
-''' - expected = '''\ -
-text -
-''' - self.assert_converts_html(input, expected, test_fun) - - def test_font_size_value_is_unsupported(self): - def test_fun(root): - convert_font_size_property_to_pt(root, 10) - return root - - input = '''\ -
-
- text -
-
''' - expected = '''\ -
-
- text -
-
''' - self.assert_converts_html(input, expected, test_fun) - - def test_simple_px(self): - def test_fun(root): - convert_font_size_property_to_pt(root, 10) - return root - - input = '''\ -
-text -
-''' - expected = '''\ -
-text -
-''' - self.assert_converts_html(input, expected, test_fun) - - def test_simple_pt(self): - def test_fun(root): - convert_font_size_property_to_pt(root, 10) - return root - - input = '''\ -
-text -
-''' - expected = '''\ -
-text -
-''' - self.assert_converts_html(input, expected, test_fun) - - def test_inherits(self): - def test_fun(root): - convert_font_size_property_to_pt(root, 10) - return root - - input = '''\ -
-
- text -
-
''' - expected = '''\ -
-
- text -
-
''' - self.assert_converts_html(input, expected, test_fun) - - def test_inherits_1em(self): - def test_fun(root): - convert_font_size_property_to_pt(root, 10) - return root - - input = '''\ -
-
- text -
- text -
-
-
''' - expected = '''\ -
-
- text -
- text -
-
-
''' - self.assert_converts_html(input, expected, test_fun) - - -class TestConvertTableBorderTopToTrBackground(HTMLTestBase): - def test_no_border(self): - def test_fun(root): - convert_table_border_top_to_tr_background(root) - return root - - input = '''\ - - - -
-
-''' - self.assert_converts_html(input, input, test_fun) - - def test_single_td(self): - def test_fun(root): - convert_table_border_top_to_tr_background(root) - return root - - input = '''\ - - - -
-
-''' - expected = '''\ - - - -
-
-''' # noqa - self.assert_converts_html(input, expected, test_fun) - - def test_multiple_td(self): - def test_fun(root): - convert_table_border_top_to_tr_background(root) - return root - - input = '''\ - - - -
- - -
-''' - expected = '''\ - - - -
- - -
-''' # noqa - self.assert_converts_html(input, expected, test_fun) - - def test_subtable_does_not_affect_parent_table(self): - def test_fun(root): - convert_table_border_top_to_tr_background(root) - return root - - input = '''\ - - - - -
- - - -
-
-
-''' - expected = '''\ - - - - -
- - - -
-
-
-''' # noqa - self.assert_converts_html(input, expected, test_fun) diff --git a/tests/test_preprocess_qch_html.py b/tests/test_preprocess_qch_html.py new file mode 100644 index 000000000..7b30d9433 --- /dev/null +++ b/tests/test_preprocess_qch_html.py @@ -0,0 +1,270 @@ +# Copyright (C) 2018 Monika Kairaityte +# Copyright (C) 2026 Cristian Adam +# +# This file is part of cppreference-doc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +import os +import tempfile +import unittest + +from lxml import etree + +from commands.preprocess_qch_html import add_color_classes +from commands.preprocess_qch_html import clamp_rowspans +from commands.preprocess_qch_html import drop_navbar_menus +from commands.preprocess_qch_html import lift_style_blocks +from commands.preprocess_qch_html import link_stylesheet +from commands.preprocess_qch_html import preprocess_html_file +from commands.preprocess_qch_html import referenced_images +from commands.preprocess_qch_html import strip_scripts +from commands.preprocess_qch_html import unstack_vertical_headers + +DATA_DIR = os.path.join(os.path.dirname(__file__), 'preprocess_qch_data') + + +class HTMLTestBase(unittest.TestCase): + def setUp(self): + self.maxDiff = None + + def parse(self, body): + return etree.fromstring( + '{0}'.format(body), etree.HTMLParser()) + + def assert_converts_html(self, input, expected_output, function): + root = self.parse(input) + function(root) + + self.assertEqual( + '{0}'.format(expected_output), + etree.tostring(root, encoding=str, method='xml')) + + +class TestLiftStyleBlocks(HTMLTestBase): + def test_takes_the_blocks_out_of_the_page(self): + root = etree.fromstring( + '' + '' + '

text

', etree.HTMLParser()) + + self.assertEqual(('.a { color: red }', '.b { color: blue }'), + lift_style_blocks(root)) + self.assertNotIn(b'.a { color: red }' + '') + + self.assertEqual(('.a { color: red }',), lift_style_blocks(root)) + + def test_ignores_an_empty_block(self): + self.assertEqual((), lift_style_blocks(self.parse(''))) + + +class TestStripScripts(HTMLTestBase): + def test_removes_them(self): + self.assert_converts_html( + '

a

', + '

a

', strip_scripts) + + +class TestDropNavbarMenus(HTMLTestBase): + def test_removes_them(self): + self.assert_converts_html( + '
C++

a

', + '

a

', drop_navbar_menus) + + def test_keeps_the_text_after_one(self): + # it belongs to the page, as it does for the