Speeding Up pdftotext When You Only Want The Text

Page extraction is naturally parallel if each worker owns its own PDFDoc.

The binary gained the thread runtime needed for page-worker pools.

One-pass XML-special detection:

static bool hasXmlSpecialScalar(const char *s, size_t len)
{
    for (size_t i = 0; i < len; ++i) {
        switch (s[i]) {
        case '&':
        case '\'':
        case '"':
        case '<':
        case '>':
            return true;
        default:
            break;
        }
    }
    return false;
}

AVX2 fast reject:

__attribute__((target("avx2")))
static bool hasXmlSpecialAvx2(const char *s, size_t len)
{
    const __m256i amp  = _mm256_set1_epi8('&');
    const __m256i apos = _mm256_set1_epi8('\'');
    const __m256i quot = _mm256_set1_epi8('"');
    const __m256i lt   = _mm256_set1_epi8('<');
    const __m256i gt   = _mm256_set1_epi8('>');

    size_t i = 0;
    for (; i + 32 <= len; i += 32) {
        const __m256i v = _mm256_loadu_si256(reinterpret_cast<const __m256i *>(s + i));
        __m256i match = _mm256_or_si256(_mm256_cmpeq_epi8(v, amp), _mm256_cmpeq_epi8(v, apos));
        match = _mm256_or_si256(match, _mm256_cmpeq_epi8(v, quot));
        match = _mm256_or_si256(match, _mm256_cmpeq_epi8(v, lt));
        match = _mm256_or_si256(match, _mm256_cmpeq_epi8(v, gt));
        if (_mm256_movemask_epi8(match)) {
            return true;
        }
    }
    return hasXmlSpecialScalar(s + i, len - i);
}

Direct append for common ASCII words:

static void appendWordText(std::string &out, const TextWord *word,
                           const UnicodeMap *uMap, bool utf8Map)
{
    if (utf8Map) {
        bool ascii = true;
        for (int i = 0; i < word->getLength(); ++i) {
            if (*word->getChar(i) >= 0x80) {
                ascii = false;
                break;
            }
        }
        if (ascii) {
            out.reserve(out.size() + word->getLength());
            for (int i = 0; i < word->getLength(); ++i) {
                out.push_back(static_cast<char>(*word->getChar(i)));
            }
            return;
        }
    }
    /* generic path */
}

Representative call-site change:

-const std::unique_ptr<std::string> wordText = word->getText();
-const std::string myString = myXmlTokenReplace(wordText->c_str());
-fprintf(f, "...>%s</word>\n", myString.c_str());
+wordText.clear();
+escapedWordText.clear();
+appendWordText(wordText, word, uMap, utf8Map);
+appendXmlTokenReplace(escapedWordText, wordText.data(), wordText.size());
+fprintf(f, "...>%s</word>\n", escapedWordText.c_str());

+static bool skipTextOnlyImageData(Stream *str, int width, int height,
+                                  int bits, int components, bool inlineImg)
+{
+    if (!inlineImg) {
+        str->close();
+        return true;
+    }
+    while (remaining > 0) {
+        const unsigned int chunk =
+            remaining > UINT_MAX ? UINT_MAX : static_cast<unsigned int>(remaining);
+        const unsigned int discarded = str->discardChars(chunk);
+        remaining -= discarded;
+        if (discarded != chunk) {
+            break;
+        }
+    }
+    str->close();
+    return true;
+}
 ...
+if (!out->needNonText()) {
+    const int components = mask ? 1 : textOnlyImageComponents(dict, csMode);
+    if (skipTextOnlyImageData(str, width, height, bits, components, inlineImg)) {
+        return;
+    }
+}

class UnorderedTextOutputDev final : public OutputDev
{
public:
    UnorderedTextOutputDev(std::string *outA, const UnicodeMap *uMapA,
                           bool utf8MapA)
        : out(outA), uMap(uMapA), utf8Map(utf8MapA) { }

    bool upsideDown() override { return true; }
    bool useDrawChar() override { return true; }
    bool interpretType3Chars() override { return false; }
    bool needNonText() override { return false; }
    bool needCharCount() override { return false; }

    void drawChar(GfxState *, double, double, double, double,
                  double, double, CharCode, int nBytes,
                  const Unicode *u, int uLen) override
    {
        if (actualText) {
            actualTextSawGlyph = actualTextSawGlyph || nBytes > 0;
            return;
        }
        if (!u || uLen <= 0) {
            return;
        }
        appendMappedText(*out, u, uLen, uMap, utf8Map, &lastWasSeparator);
    }

    void endString(GfxState *) override
    {
        if (!out->empty() && !lastWasSeparator) {
            appendMappedUnicode(*out, 0x20, uMap, utf8Map);
            lastWasSeparator = true;
        }
    }

    void beginActualText(GfxState *, const std::string &text) override
    {
        actualText = text;
        actualTextSawGlyph = false;
    }

    void endActualText(GfxState *) override
    {
        if (actualText && actualTextSawGlyph) {
            const std::vector<Unicode> uText = TextStringToUCS4(*actualText);
            appendMappedText(*out, uText.data(), static_cast<int>(uText.size()),
                             uMap, utf8Map, &lastWasSeparator);
        }
        actualText.reset();
        actualTextSawGlyph = false;
    }
};

static bool renderUnorderedPagesParallel(const GooString &fileName,
                                         const std::optional<GooString> &ownerPW,
                                         const std::optional<GooString> &userPW,
                                         int first, int last, int nThreads,
                                         EndOfLineKind textEOL, FILE *f)
{
    std::atomic<int> nextPage{ first };
    std::atomic<bool> failed{ false };
    std::mutex writeMutex;
    std::vector<std::thread> workers;
    workers.reserve(nThreads);

    for (int i = 0; i < nThreads; ++i) {
        workers.emplace_back([&]() {
            std::unique_ptr<PDFDoc> workerDoc =
                PDFDocFactory().createPDFDoc(fileName, ownerPW, userPW);
            if (!workerDoc || !workerDoc->isOk()) {
                failed.store(true);
                return;
            }

            std::string pageOutput;
            while (!failed.load()) {
                const int page = nextPage.fetch_add(1);
                if (page > last) {
                    break;
                }
                if (!renderUnorderedPage(workerDoc.get(), page, textEOL,
                                         &pageOutput)) {
                    failed.store(true);
                    break;
                }
                std::lock_guard<std::mutex> lock(writeMutex);
                fwrite(pageOutput.data(), 1, pageOutput.size(), f);
            }
        });
    }
    ...
}