Q: Java 1.7 Subject Hash of X.509 Certificate OpenSSL 1.0+ Compatible I've been struggling with this for a couple of days. I'm working on a Java 1.7 app running in an embedded Linux environment. OpenSSL is not available and I don't have control over what is in the OS image on the device. I need to compute the subject hash of a self-signed X.509 certificate, producing the same result as OpenSSL 1.0+. This existing answer got me started:
The new subject hash openssl algorithm differs
The code from my test app appears below. My computation works for certs whose subject name contains only a CN value, but it does not work for certs with any other subject component (OU, O, L, ST, or C) specified. For those certs the hash on the entire subject (less the intro sequence) does not match. Per the above answer, I have extracted each component (using the getObjectAt( ) method) and hashed each of them them alone (no joy), reversed their order and hashed them all (no joy), and a number of other variations on that theme. I have been trying to avoid what I fear will be the more time-consuming effort of downloading the OpenSSL source and getting it running so I can examine intermediate results and see where I am going wrong. Perhaps someone who has done this can provide some guidance.
private static void getSubjectHash( X509Certificate x509Cert )
{
    try {
        // get the subject principal
        X500Principal x500Princ = x509Cert.getSubjectX500Principal( );

        // create a new principal using canonical name (order, spacing, etc.) and get it in ANS1 DER format
        byte[] newPrincEnc = new X500Principal( x500Princ.getName( X500Principal.CANONICAL ) ).getEncoded( );

        // read it in as an ASN1 Sequence to avoid custom parsing
        ASN1InputStream aIn = new ASN1InputStream( newPrincEnc );
        ASN1Sequence seq = (ASN1Sequence) aIn.readObject( );

        List<byte[]> terms = new ArrayList<>( );
        int finalLen = 0;
        int i = 0;

        // hash the encodables for each term individually and accumulate them in a list
        for ( ASN1Encodable asn1Set : seq.toArray( ) ) {
            byte[] term = ( (ASN1Set) asn1Set ).getEncoded( );
            terms.add( term );
            finalLen += term.length;

            // digest the term
            byte[] hashBytes = truncatedHash( getDigest( term ), 4 );
            printByteArray( String.format( "hash of object at %d:", i++ ), hashBytes );

            System.out.println( "" );
        }


        // hash all terms together in order of appearance
        int j = 0;
        byte[] finalEncForw = new byte[finalLen];
        for ( byte[] term : terms )
            for ( byte b : term )
                finalEncForw[j++] = b;

        // digest and truncate
        byte[] hashBytes = truncatedHash( getDigest( finalEncForw ), 4 );

        printByteArray( "hash of all terms in forward order", hashBytes );
        System.out.println( "" );


        // hash all terms together in reverse order
        j = 0;
        byte[] finalEncRev = new byte[finalLen];
        for ( int k = terms.size( ) - 1; k >= 0; --k )
            for ( byte b : terms.get( k ) )
                finalEncRev[j++] = b;

        // digest and truncate
        hashBytes = truncatedHash( getDigest( finalEncRev ), 4 );

        printByteArray( "hash of all terms in reverse order", hashBytes );
    }
    catch ( Exception ex ) {
        throw new RuntimeException( "uh-oh" );
    }
}

private static byte[] getDigest( byte[] toHash )
{
    MessageDigest md;

    try {
        md = MessageDigest.getInstance( "SHA1" );
    }
    catch ( NoSuchAlgorithmException nsa ) {
        throw new RuntimeException( "no such algorithm" );
    }

    return md.digest( toHash );
}

private static byte[] truncatedHash( byte[] hash, int truncatedLength )
{
    if ( truncatedLength < 1 || hash.length < 1 )
        return new byte[0];

    byte[] result = new byte[truncatedLength];

    for ( int i = 0; i < truncatedLength; ++i )
        result[truncatedLength - 1 - i] = hash[i];

    return result;
}

private static void printByteArray( String name, byte[] bytes )
{
    System.out.println( name + " length=" + String.valueOf( bytes.length ) );
    for ( byte b: bytes ) {
        System.out.print( String.format( "%02X ", Byte.toUnsignedInt( b ) ) );
    }

    System.out.println( );
}


A: Okay, duct tape it is, for now. This appears to work for all the certs I have available to test. This is the re-written version of the getSubjectHash method:
private static void getSubjectHash( X509Certificate x509Cert )
{
    try {
        // get the subject principal
        X500Principal x500Princ = x509Cert.getSubjectX500Principal( );

        // create a new principal using canonical name (order, spacing, etc.) and get it in ANS1 DER format
        byte[] newPrincEnc = new X500Principal( x500Princ.getName( X500Principal.CANONICAL ) ).getEncoded( );

        // read it in as an ASN1 Sequence to avoid custom parsing
        ASN1InputStream aIn = new ASN1InputStream( newPrincEnc );
        ASN1Sequence seq = (ASN1Sequence) aIn.readObject( );

        List<byte[]> terms = new ArrayList<>( );
        int finalLen = 0;
        int i = 0;

        // hash the encodables for each term individually and accumulate them in a list
        for ( ASN1Encodable asn1Set : seq.toArray( ) ) {
            byte[] term = ( (ASN1Set) asn1Set ).getEncoded( );
            term[9] = 0x0c; // change tag from 0x13 (printable string) to 0x0c
            terms.add( term );
            finalLen += term.length;

            // digest the term
            byte[] hashBytes = truncatedHash( getDigest( term ), 4 );
            printByteArray( String.format( "hash of object at %d:", i++ ), hashBytes );

            System.out.println( "" );
        }


        // hash all terms together in order of appearance
        int j = 0;
        byte[] finalEncForw = new byte[finalLen];
        for ( byte[] term : terms )
            for ( byte b : term )
                finalEncForw[j++] = b;

        // digest and truncate
        byte[] hashBytes = truncatedHash( getDigest( finalEncForw ), 4 );

        printByteArray( "hash of all terms in forward order", hashBytes );
        System.out.println( "" );

    }
    catch ( Exception ex ) {
        throw new RuntimeException( "uh-oh" );
    }
}


A: This answer was the closest I've found to a good one, but it's far from that
There are a few misconceptions in it
- X509_NAME_hash returns a long unsigned long X509_NAME_hash(X509_NAME *x)
- it's not the 10th character that needs to be 0x0c, rather the 1st character of the value
- converting the text to lower case does not solve the problem  
In order to overcome these, I've started using the X500Name instead of the X500Principal as an input parameter a conversion between the two can be done easily
The reason for this is that the X500Name exposes the RDN array from where we can retrieve the values (here, i'm ignoring the multi value options and only using the first)
retrieving the name allows me to do canonical conversion (not only lowercase) and to know where it start, to replace the 1st byte to 0x0c 
Updated code now includes the full solution without multiple byte to string and back conversions
public static long calculateX500NameHash(X500Name name) throws IOException, NoSuchAlgorithmException {
    byte[] nameEncoded = name.getEncoded();

    final ASN1Sequence asn1Sequence = (ASN1Sequence) ASN1Primitive.fromByteArray(nameEncoded);
    List<byte[]> rdnList = new ArrayList<>();
    int length = 0;
    for (ASN1Encodable asn1Set : asn1Sequence.toArray()) {
        byte[] bytes = ((ASN1Set) asn1Set).getEncoded();
        length += bytes.length;
        rdnList.add(bytes);
    }
    byte[] nameBytes = new byte[length];
    int counter = 0;
    int addedItems = 0;
    for (RDN rdn : name.getRDNs()) {
        // Get original encoded RDN value
        byte[] encoded = rdn.getFirst().getValue().toASN1Primitive().getEncoded();
        // Get the RDN value as string without the prefix
        StringBuilder content = new StringBuilder();
        for (int j = 2; j < encoded.length; j++) {
            content.append((char) encoded[j]);
        }
        // canonicalize the string
        byte[] updateContent = IETFUtils.canonicalize(content.toString()).getBytes(StandardCharsets.UTF_8);
        // create new byte[] with the updated prefix and canonicalized string
        byte[] updated = new byte[encoded.length];
        updated[0] = 0x0c;
        updated[1] = encoded[1];
        System.arraycopy(updateContent, 0, updated, 2, updateContent.length);
        // get full RDN with type prefix
        byte[] rdnFromList = rdnList.get(counter);
        int fullLength = rdnFromList.length;
        int valueLength = encoded.length;
        // Additional check, expect to always return true
        if (isMatchingTheEnd(rdnFromList, encoded)) {
            int prefixLength = (fullLength - valueLength);
            // add the beginning of the full RDN to the `nameBytes` array without the value
            System.arraycopy(rdnFromList, 0, nameBytes, addedItems, prefixLength);
            // add the updated value to the `nameBytes` array
            System.arraycopy(updated, 0, nameBytes, addedItems + prefixLength, valueLength);
        } else {
            // safeguard
            System.arraycopy(rdnFromList, 0, nameBytes, addedItems, fullLength);
        }
        addedItems += fullLength;
        ++counter;
    }
    return getHashFromByteArray(nameBytes) & 0xffffffffL;
}

private static boolean isMatchingTheEnd(byte[] fullRdn, byte[] rdnValue) {
    int fullRdnLength = fullRdn.length;
    int rdnValueLength = rdnValue.length;
    if (fullRdnLength > rdnValueLength) {
        int prefixLength = fullRdnLength - rdnValueLength;
        for (int i = 0; i < rdnValueLength; i++) {
            if (fullRdn[prefixLength + i] != rdnValue[i]) {
                return false;
            }
        }
        return true;
    }
    return false;
}


private static long getHashFromByteArray(byte[] nameBytes) throws NoSuchAlgorithmException {
    byte[] digest = MessageDigest.getInstance("SHA1").digest(nameBytes);
    return (((digest[0] & 0xff))
            | (((digest[1] & 0xff) << 8))
            | (((digest[2] & 0xff) << 16))
            | (((digest[3] & 0xff) << 24)));
}

Hope this helps someone

A: Several answers here were given, they were extremely helpful for me, I claim them to be incomplete (wrong). Forget the canoncial format created by Java, it is not compatible with the one created by OpenSSL and it cannot be used to recode to OpenSSL format. Please note, there is no standard for a "canonical" format. (I can go in detail if required).
My code is based on the RFC's Name definition definition (not covered by other answers) and the OpenSSL code (not fully covered by other answers).
I have tested my coded against:

*

*https://curl.haxx.se/ca/cacert.pem

*https://www.quovadisglobal.com/download-roots-crl/

*https://new.siemens.com/global/en/general/legal/ca-certificates.html
Verification in C:
#include <openssl/asn1.h>
#include <stdio.h>
#include <string.h>


int main(void) {
  ASN1_STRING * tugra_asn1 = ASN1_STRING_type_new(V_ASN1_UTF8STRING);
  /*char *tugra = "E-Tuğra EBG Bilişim Teknolojileri ve Hizmetleri A.Ş.";
   */
  char *wikipedia = "Википедия";
  ASN1_STRING_set(tugra_asn1, tugra, -1);
  printf("ASN1_STRING_length: %d\n", ASN1_STRING_length(tugra_asn1));
  ASN1_STRING * tugra_asn1_canon = ASN1_STRING_new();
  int ret = asn1_string_canon(tugra_asn1_canon, tugra_asn1);
  printf("ret: %d\n", ret);
  printf("ASN1_STRING_length: %d\n", ASN1_STRING_length(tugra_asn1_canon));
  const unsigned char * data = ASN1_STRING_data(tugra_asn1_canon);
  printf("ASN1_STRING_canon: %s\n", data);

  printf("ASN1_tag2str: %s\n", ASN1_tag2str(ASN1_STRING_type(tugra_asn1)));
  return 0;

}

The code works with OpenSSL 1.0.2+, but needs to be modified because asn1_string_canon is static. Remove and recompile OpenSSL.
Now the Java code:
    byte[] encoded = subject.getEncoded();

    Asn1Sequence asn1Name = (Asn1Sequence) Asn1.decode(encoded);
    ByteBuffer recoded = ByteBuffer.allocate(asn1Name.getContainer().getBodyLength());

    // Based on https://github.com/openssl/openssl/blob/852c2ed260860b6b85c84f9fe96fb4d23d49c9f2/crypto/x509/x_name.c#L296-L306
    // We only need the sequence elements
    for (Asn1Type asn1type0 : asn1Name.getValue()) {
      Asn1Set asn1Rdn = (Asn1Set) asn1type0;
      for (Asn1Type asn1type1 : asn1Rdn.getValue()) {
        Asn1Sequence asn1Ava = (Asn1Sequence) asn1type1;
        List<Asn1Type> asn1AvaTV = asn1Ava.getValue();
        Asn1ObjectIdentifier asn1AttrType = (Asn1ObjectIdentifier) asn1AvaTV.get(0);
        Asn1Type asn1AttrValue = asn1AvaTV.get(1);
        UniversalTag valueTag = asn1AttrValue.tag().universalTag();
        switch(valueTag) {
        case UTF8_STRING:
        case BMP_STRING:
        case UNIVERSAL_STRING:
        case PRINTABLE_STRING:
        case T61_STRING:
        case IA5_STRING:
        case VISIBLE_STRING:
          Asn1String asn1AttrValueString = (Asn1String) asn1AttrValue;
          String string = asn1AttrValueString.getValue();
          string = string.replaceAll("^\\s+|\\s+$", "").replaceAll("\\s+", " ");
          char[] chars = string.toCharArray();
          for (int i = 0; i < chars.length; i++) {
            char c = chars[i];
            if (c >= 'A' && c <= 'Z')
              chars[i] = Character.toLowerCase(c);
          }
          String utf8String = new String(chars);
          Asn1Utf8String asn1Utf8Sring = new Asn1Utf8String(utf8String);
          asn1AttrValue = asn1Utf8Sring;

          asn1Ava.clear();
          asn1Ava.addItem(asn1AttrType);
          asn1Ava.addItem(asn1AttrValue);
          break;
        default:
          // leave as-is
          break;
        }
      }

      byte[] asn1RdnDer = asn1Rdn.encode();

      // Concat for hash
      if (recoded.position() + asn1RdnDer.length > recoded.capacity()) {
        ByteBuffer tmp = recoded;
        recoded = ByteBuffer.allocate(tmp.position() + asn1RdnDer.length);
        tmp.flip();
        recoded.put(tmp);
      }
      recoded.put(asn1RdnDer);
    }

    recoded.flip();

    try {
      MessageDigest sha1 = MessageDigest.getInstance("SHA-1");
      byte[] hash = sha1.digest(Arrays.copyOf(recoded.array(), recoded.remaining()));

      int truncHash = (((hash[0] & 0xff)) | (((hash[1] & 0xff) << 8))
          | (((hash[2] & 0xff) << 16)) | (((hash[3] & 0xff) << 24)));
      System.out.printf("subject hash: %08x%n", truncHash);
    } catch (NoSuchAlgorithmException e) {
      // Should not happen for SHA-1
    }

You will need:
<dependency>
  <groupId>org.apache.kerby</groupId>
  <artifactId>kerby-asn1</artifactId>
  <version>2.0.1-SNAPSHOT</version>
</dependency>

A very lightweight ASN.1 library (60 kB), way less than BC.
Here is a self-signed cert with UTF-8 bytes as well as a lot of whitespace:
-----BEGIN CERTIFICATE-----
MIIGzTCCBLWgAwIBAgIUAVhZJ/kW56acy4DEfDSK/kwP/kQwDQYJKoZIhvcNAQEL
BQAwgfUxCzAJBgNVBAYTAkRFMRYwFAYDVQQIDA0gIELDtnIgbGluICAgMRwwGgYD
VQQHDBMgIELDllIgbCAgICAgaU4gICAgMTYwNAYDVQQKDC0gINCS0LjQutC40L/Q
tdC00LjRjiAgINCS0LjQutC40L/QtdC00LjRjiAgICAxHTAbBgNVBAsMFEV4YW1w
bGUgICAgQ29ycC4gICAgMTMwMQYDVQQDDCogICBNaWNoYWVsLU8gICBDZXJ0aWZp
Y2F0ZSAgIEF1dGhvcml0eSAgICAxJDAiBgkqhkiG9w0BCQEWFU1JQ0hBRUwtT0BF
WEFNUExFLkNPTTAeFw0yMDA1MTQyMjQ4MTVaFw0yMzAyMDgyMjQ4MTVaMIH1MQsw
CQYDVQQGEwJERTEWMBQGA1UECAwNICBCw7ZyIGxpbiAgIDEcMBoGA1UEBwwTICBC
w5ZSIGwgICAgIGlOICAgIDE2MDQGA1UECgwtICDQktC40LrQuNC/0LXQtNC40Y4g
ICDQktC40LrQuNC/0LXQtNC40Y4gICAgMR0wGwYDVQQLDBRFeGFtcGxlICAgIENv
cnAuICAgIDEzMDEGA1UEAwwqICAgTWljaGFlbC1PICAgQ2VydGlmaWNhdGUgICBB
dXRob3JpdHkgICAgMSQwIgYJKoZIhvcNAQkBFhVNSUNIQUVMLU9ARVhBTVBMRS5D
T00wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC1T9Ng17hOj4GKrf2Z
/ug30RMimYyjgb++sJeOl7p3sSuCHuorKEGNW53VA+eL3sr6y/adR89ZxqSxXMlw
iNWVe40NXlSr9YiYBzO0Xl1Lze5Gjb+LkDWECrTAyjplJh/ru2uKa3vje7GFwA5z
alT2Qes8EBQ0za2aKP1Uwj5de1YRr1djxl2HVqxN7ReihV9ecB7++5zSNMzqhM0t
uc7VFljY6n49cPn0zDzaZCcbCQ7EII8Jt6hGLLJKCwzofPQ4keX6UxC203nXOP7S
w63XaSbymnXgC6I6IohsCogv4c3DKh4v/h73Ai4ya/iVSLCCbaHIrIkUhnU7fyGU
VOT+KoCCGbqXam9kW01GGNui+JvT6wAraiKZLnfzT/lHI0qbjAB9wzvhur74C9Pv
fLlg5TVzBN3s3oTNjZvI87bRoipANlOUy4GfX/NxMQdCVvMaHdMl5VztlttwK2I6
flSiYm97rdDSrSmPuvp36/7QYXE2+Zzf+34rRrxhb5LeN3ltA9Gy9U5a3ANaCBqs
C94TdKX59qavDN5Usml3hgvz8oTLPXJ/YPqxAEsxzSyEPEc7/ywEespEz/YfeuLe
eOuL1s8nOiBOOuHVphtH1LmjvTRX+tOv7uf65nqiwKH98pU0Y+F+1gIpsCgYN7s1
4jc7iCeIVinwTT0Kfs8L+KpgIwIDAQABo1MwUTAdBgNVHQ4EFgQUChh34sOcSjBJ
PP4/3zYK0Z16wtswHwYDVR0jBBgwFoAUChh34sOcSjBJPP4/3zYK0Z16wtswDwYD
VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAK7OPbrfYMJgmZlhwtiWb
v5pyWvOF5Py3bY2PFr17cGtWiV7QFcE/PG8mN3+2WkbL4q7iNsOO8xMiGfDLJdtD
QruNAL2loGatpTl0TXZtgPzW3fx7HG1NrQH1fIjgGj9DgDrou3AVIoaYmJNgk+HT
jj4K3rC7RbnLkkKYGCwAYn3GRQPfGgQL3nJjn4ajd3JoaZbsfN0iEUevX4DmLfMn
sKPVaLRvNLbWGEs7ZiRC6ZRbncg3GVeOVO6WZuUknaePHyWaO+5tgZyi9GnywPAZ
qdjNvme+tzm2En3Jw1N9CTDd9SNmHK5Fd17fZp6Qa0LdSJQddNKxzhho01klPL+Q
N7DfjUKD9/LHy9KCeTwKMqLGIDlYSuuKx7KEyrVe749zVe9FGBuyxxsb5cukE3zx
q+S1HP9+RdKZYavmZ9+WrW8i/S0PpE8t5ZgeRCUz9SseGewZ2W2aeGiquJCBj/vz
+5iSOIEN8lw58+FGGrLrEBQQlNSVkDleEFR3wV8ww1vBLp1mhyPnPilDI7N7tfWW
kOvoS860lKN9jlXeyPdMd/aDrrBptiewZHxgxtgTV55ubJuL2l4Q52ZBAXE6cR/p
PWehO0gzBik6f4aekDCgPt9zFiCiQNN8p8yyFUQ4mJsW6MZaGB0rJUUWyx2jT4F6
n0tEnfE7rodFIjuSFxBSD2k=
-----END CERTIFICATE-----

Subject: emailAddress=MICHAEL-O@EXAMPLE.COM,CN=\ \ \ Michael-O   Certificate   Authority\ \ \ \ ,OU=Example    Corp.\ \ \ \ ,O=\ \ Википедию   Википедию\ \ \ \ ,L=\ \ BÖR l     iN\ \ \ \ ,ST=\ \ Bör lin\ \ \ ,C=DE
Fingerprint (SHA-256): F0:04:0D:38:8A:E5:93:A8:51:1D:06:3E:96:8F:44:29:29:F2:2D:57:A1:5F:7B:CB:F9:F4:EE:98:5B:A8:50:CA
Subject Hash: 5ba4b7de
Properly canonicalized subject X.509 name from my Java code in DER: MIHMMQswCQYDVQQGDAJkZTERMA8GA1UECAwIYsO2ciBsaW4xEjAQBgNVBAcMCWLDlnIgbCBpbjEuMCwGA1UECgwl0JLQuNC60LjQv9C10LTQuNGOINCS0LjQutC40L/QtdC00LjRjjEWMBQGA1UECwwNZXhhbXBsZSBjb3JwLjEoMCYGA1UEAwwfbWljaGFlbC1vIGNlcnRpZmljYXRlIGF1dGhvcml0eTEkMCIGCSqGSIb3DQEJAQwVbWljaGFlbC1vQGV4YW1wbGUuY29t

A: Thanks for your code dude. I've improved it for supporting certificates that contains extended ASCII characters in the subject (éËÁñç for example).
public static int X509_NAME_hash(X509Certificate x509Cert) throws IOException, NoSuchAlgorithmException {
    // get the subject principal
    X500Principal x500Princ = x509Cert.getSubjectX500Principal();
    byte[] newPrincEnc = x500Princ.getEncoded();
    final ASN1Sequence asn1Sequence = (ASN1Sequence) ASN1Primitive.fromByteArray(newPrincEnc);

    Debugger.log(asn1Sequence);

    List<byte[]> terms = new ArrayList<>();
    int finalLen = 0;

    // hash the encodables for each term individually and accumulate them in a list
    for (ASN1Encodable asn1Set : asn1Sequence.toArray()) {
        byte[] term = ((ASN1Set) asn1Set).getEncoded();
        term[9] = 0x0c; // change tag from 0x13 (printable string) to 0x0c

        for (int i = 11; i < term.length; i++) {
            byte actual = term[i];
            //lowercase only if the character is not ASCCI Extended (below 126)
            if (actual < 127) {
                term[i] = (byte) Character.toLowerCase((char) actual);
            }
        }

        terms.add(term);
        finalLen += term.length;
    }

    // hash all terms together in order of appearance
    int j = 0;
    byte[] finalEncForw = new byte[finalLen];
    for (byte[] term : terms)
        for (byte b : term)
            finalEncForw[j++] = b;

    return peekInt(MessageDigest.getInstance("SHA1").digest(finalEncForw), 0, ByteOrder.LITTLE_ENDIAN);
}

public static X509Certificate readCertificate(File rootFile) throws CertificateException, IOException {
    CertificateFactory fact = CertificateFactory.getInstance("X.509");
    FileInputStream is = new FileInputStream(rootFile);
    return (X509Certificate) fact.generateCertificate(is);
}

public static int peekInt(byte[] src, int offset, ByteOrder order) {
    if (order == ByteOrder.BIG_ENDIAN) {
        return (((src[offset++] & 0xff) << 24) | ((src[offset++] & 0xff) << 16) | ((src[offset++] & 0xff) << 8)
                | ((src[offset] & 0xff) << 0));
    } else {
        return (((src[offset++] & 0xff) << 0) | ((src[offset++] & 0xff) << 8) | ((src[offset++] & 0xff) << 16)
                | ((src[offset] & 0xff) << 24));
    }
}

