RSA signing between PHP and C++ using phpseclib and Crypto++ and PSS and SHA256

app.cpp

// use https://github.com/noloader/cryptopp-pem to add pem loading capabilities to Crypto++
// On Windows those must be manually added to the project and compiled

void readKeys( << INPUTS >> ) {
    RSA::PrivateKey privateKey;
    RSA::PublicKey publicKey;

    qDebug() << "Loading pub/priv key from file";

    FileSource privatefs("C:\\del\\dpimkey2", true);
    PEM_Load(privatefs, privateKey);

    FileSource publicfs("C:\\del\\dpimpubkey2.out", true);
    PEM_Load(publicfs, publicKey);
}

QString encrypt( << HERE GO INPUTS >> ) {
  try {
    CryptoPP::AutoSeededRandomPool rng;

    QString hexPublic = ui->rsaPublicKeyInput->text();
    RSA::PublicKey publicKey = rsaKeyFromHex<RSA::PublicKey>(hexPublic);

    std::string plain, cipher;
    plain = ui->rsaPlainTextInput->toPlainText().toStdString();

    qDebug() << "plain size" << plain.size();

    // using SHA256 because phpseclib is using it
    RSAES<OAEP<SHA256> >::Encryptor e(publicKey);

    StringSource(plain, true,
               new PK_EncryptorFilter(rng, e,
                                      new HexEncoder(
                                          new StringSink(cipher)
                                          )
                                      )
               );

    QString hexencrypted = QString::fromStdString(cipher);
    ui->rsaEncryptedTextInput->setPlainText(hexencrypted);

  } catch(const Exception& ex) {
    qDebug() << ex.what();
    QMessageBox box;
    box.setText(ex.what());
    box.exec();
  }
}

QString decrypt( << HERE GO INPUTS >> ) {
  
  
  try {
    CryptoPP::AutoSeededRandomPool rng;

    QString hexPrivate = ui->rsaPrivateKeyInput->text();
    RSA::PrivateKey privateKey = rsaKeyFromHex<RSA::PrivateKey>(hexPrivate);

    std::string cipher, decrypted_data;
    cipher = ui->rsaEncryptedTextInput->toPlainText().toStdString();

    // using SHA256 because phpseclib is using it
    RSAES<OAEP<SHA256> >::Decryptor d(privateKey);
    StringSource(cipher, true,
                 new HexDecoder(
                     new PK_DecryptorFilter(rng, d,
                                            new StringSink(decrypted_data)
                                            )
                     )
                 );

    QString decrstr = QString::fromStdString(decrypted_data);
    ui->rsaPlainTextInput->setPlainText(decrstr);

  } catch(const Exception& ex) {
    qDebug() << ex.what();
    QMessageBox box;
    box.setText(ex.what());
    box.exec();
  }
  
}

void verifySignature( << HERE GO INPUTS >> ) {

  RSA::PublicKey publicKey;
  FileSource publicfs("C:\\del\\dpimpubkey2.out", true);
  PEM_Load(publicfs, publicKey);

  // because backend phpseclib is using PSS with SHA256
  RSASS<PSS, SHA256>::Verifier verifier(publicKey);


  QString signaturehex = ui->signatureInput->toPlainText();
  // std string works
  std::string signature = QByteArray::fromHex(signaturehex.toLatin1()).toStdString();
  // QByteArray does not work!!! QByteArray usually does not play well with Crypto++
  // QByteArray signature = decodehex(signaturehex.toLatin1());

    bool result = verifier.VerifyMessage((const byte*) &message[0], message.size(),
                                         (const byte*) &signature[0], signature.size());
    if (result) {
      ui->signResultLbl->setText("VERFIFIED");
    } else {
      ui->signResultLbl->setText("NOT VERIFIED");
    }
}

gen_keys.txt

1. install openssl

2. generate key

ssh-keygen -t rsa

it will output two files, pirvate and public

3. use openssl to generate public files

openssl rsa -in private_key_filename -pubout -outform PEM -out public_key_output_filename

4. use original private key, and generated key as a public key

e.g.
ssh-keygen -t rsa
Enter file in which to save the key (C:\Users\edin-m/.ssh/id_rsa): C:\del\dpimkey2
C:\del>openssl rsa -in dpimkey2 -pubout -outform PEM -out dpimpubkey2.out
C:\del>ls dpimkey2 dpimpubkey2.out
dpimkey2  dpimpubkey2.out

rsa.php

<?php
include('vendor/autoload.php');

use phpseclib3\Crypt\RSA;

$privateKey = RSA::load(file_get_contents("C:\\del\\dpimkey2"));
$publicKey = RSA::load(file_get_contents("C:\\del\\dpimpubkey2.out"));


$message = "wicked sick";

$cipher = $publicKey->withPadding(RSA::ENCRYPTION_OAEP)->encrypt($message);
echo 'cipher: '.strtoupper(bin2hex($cipher)).'<br>';

$text = $privateKey->withPadding(RSA::ENCRYPTION_OAEP)->decrypt($cipher);
echo 'decipher: '.$text.'<br>';

$privateKey = $privateKey->withPadding(RSA::SIGNATURE_PSS);
$signature = $privateKey->withPadding(RSA::SIGNATURE_PSS)->sign($message);
echo 'signature: '.strtoupper(bin2hex($signature)).'<br>';

$isvalid = $publicKey->withPadding(RSA::SIGNATURE_PSS)->verify($message, $signature) ? 'valid' : 'invalid';
echo 'is valid: '.$isvalid.'<br>';

?>