test.cc

/** @file
 * @brief Simple utility to demonstrate an issue with fullwidth characters
 *
 */
/* Based on simplesearch.cc:
 *
 * Copyright (C) 2007-2022 Olly Betts
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 */

#ifdef HAVE_CONFIG_H
# include <config.h>
#endif

#include <xapian.h>

#include <iostream>
#include <string>

#include <cstdlib> // For exit().
#include <cstring>

using namespace std;

int
main(int argc, char **argv) {
try {
    if (argc != 2) {
        int rc = 1;
        cout << "Usage: " << argv[0] << " PATH_TO_DATABASE\n";
        exit(rc);
    }

    {
        // Open the database for writing
        Xapian::WritableDatabase db(argv[1], Xapian::DB_CREATE_OR_OVERWRITE);
        Xapian::TermGenerator indexer;
        indexer.set_flags(Xapian::TermGenerator::FLAG_WORD_BREAKS);
        Xapian::Document doc;
        indexer.set_document(doc);
        string text("三菱ＵＦＪファクター株式会社");
        cout << "Indexing text: " << text << '\n';
        indexer.index_text(text);
        db.add_document(doc);
    }

    {
        // Open the database for searching.
        Xapian::Database db(argv[1]);
        for (auto it = db.allterms_begin(); it != db.allterms_end(); ++it) {
                cout << "Database term: " << *it << '\n';
        }
        cout << '\n';

        for (std::string query : { "ＵＦＪ", "三菱ＵＦＪファクター株式会社" }) {
            // Parse the query string to produce a Xapian::Query object.
            Xapian::QueryParser qp;
            qp.set_database(db);
            cout << "Querying for: " << query << '\n';
            Xapian::Query q = qp.parse_query(query, Xapian::QueryParser::FLAG_WORD_BREAKS);
            cout << "Parsed query is: " << q.get_description() << '\n';

            // Start an enquire session.
            Xapian::Enquire enquire(db);
            enquire.set_query(q);
            Xapian::MSet matches = enquire.get_mset(0, 10);
            
            // Display the results.
            cout << matches.get_matches_estimated() << " results found.\n";
            cout << "Matches 1-" << matches.size() << ":\n\n";
        }
    }
} catch (const Xapian::Error &e) {
        cout << e.get_description() << '\n';
        exit(1);
}
}

This produces

Indexing text: 三菱ＵＦＪファクター株式会社
Database term: ファクター
Database term: 三菱
Database term: 株式会社
Database term: ＵＦＪ

Querying for: ＵＦＪ
Parsed query is: Query(ｕｆｊ@1)
0 results found.
Matches 1-0:

Querying for: 三菱ＵＦＪファクター株式会社
Parsed query is: Query((三菱@1 AND ｕｆｊ@1 AND ファクター@1 AND 株式会社@1))
0 results found.
Matches 1-0:

Note how using FLAG_WORD_BREAKS in the query-parser causes it to lowercase the fullwidth ＵＦＪ search term during query, although it got indexed in capital letters.