Change SHA2 implementation based on OpenSSL to use EVP digest routines

The use of low-level hash routines is not recommended by upstream
OpenSSL since 2000, and pgcrypto already switched to EVP as of 5ff4a67.
Note that this also fixes a failure with SCRAM authentication when using
FIPS in OpenSSL, but as there have been few complaints about this
problem and as this causes an ABI breakage, no backpatch is done.

Author: Michael Paquier, Alessandro Gherardi
Reviewed-by: Daniel Gustafsson
Discussion: https://postgr.es/m/20200924025314.GE7405@paquier.xyz
Discussion: https://postgr.es/m/20180911030250.GA27115@paquier.xyz

Author: michaelpq

src/common/sha2_openssl.c

@@ -20,83 +20,118 @@
 #include "postgres_fe.h"
 #endif
 
-#include <openssl/sha.h>
-
 #include "common/sha2.h"
 
+#ifdef FRONTEND
+#include "common/logging.h"
+#else
+#include "miscadmin.h"
+#endif
+
+#ifdef FRONTEND
+#define sha2_log_and_abort(...) \
+	do { pg_log_fatal(__VA_ARGS__); exit(1); } while(0)
+#else
+#define sha2_log_and_abort(...) elog(ERROR, __VA_ARGS__)
+#endif
+
+static void
+digest_init(EVP_MD_CTX **ctx, const EVP_MD *type)
+{
+	*ctx = EVP_MD_CTX_create();
+	if (*ctx == NULL)
+		sha2_log_and_abort("could not create EVP digest context");
+	if (EVP_DigestInit_ex(*ctx, type, NULL) <= 0)
+		sha2_log_and_abort("could not initialize EVP digest context");
+}
+
+static void
+digest_update(EVP_MD_CTX **ctx, const uint8 *data, size_t len)
+{
+	if (EVP_DigestUpdate(*ctx, data, len) <= 0)
+		sha2_log_and_abort("could not update EVP digest context");
+}
+
+static void
+digest_final(EVP_MD_CTX **ctx, uint8 *dest)
+{
+	if (EVP_DigestFinal_ex(*ctx, dest, 0) <= 0)
+		sha2_log_and_abort("could not finalize EVP digest context");
+	EVP_MD_CTX_destroy(*ctx);
+}
 
 /* Interface routines for SHA-256 */
 void
 pg_sha256_init(pg_sha256_ctx *ctx)
 {
-	SHA256_Init((SHA256_CTX *) ctx);
+	digest_init(ctx, EVP_sha256());
 }
 
 void
 pg_sha256_update(pg_sha256_ctx *ctx, const uint8 *data, size_t len)
 {
-	SHA256_Update((SHA256_CTX *) ctx, data, len);
+	digest_update(ctx, data, len);
 }
 
 void
 pg_sha256_final(pg_sha256_ctx *ctx, uint8 *dest)
 {
-	SHA256_Final(dest, (SHA256_CTX *) ctx);
+	digest_final(ctx, dest);
 }
 
 /* Interface routines for SHA-512 */
 void
 pg_sha512_init(pg_sha512_ctx *ctx)
 {
-	SHA512_Init((SHA512_CTX *) ctx);
+	digest_init(ctx, EVP_sha512());
 }
 
 void
 pg_sha512_update(pg_sha512_ctx *ctx, const uint8 *data, size_t len)
 {
-	SHA512_Update((SHA512_CTX *) ctx, data, len);
+	digest_update(ctx, data, len);
 }
 
 void
 pg_sha512_final(pg_sha512_ctx *ctx, uint8 *dest)
 {
-	SHA512_Final(dest, (SHA512_CTX *) ctx);
+	digest_final(ctx, dest);
 }
 
 /* Interface routines for SHA-384 */
 void
 pg_sha384_init(pg_sha384_ctx *ctx)
 {
-	SHA384_Init((SHA512_CTX *) ctx);
+	digest_init(ctx, EVP_sha384());
 }
 
 void
 pg_sha384_update(pg_sha384_ctx *ctx, const uint8 *data, size_t len)
 {
-	SHA384_Update((SHA512_CTX *) ctx, data, len);
+	digest_update(ctx, data, len);
 }
 
 void
 pg_sha384_final(pg_sha384_ctx *ctx, uint8 *dest)
 {
-	SHA384_Final(dest, (SHA512_CTX *) ctx);
+	digest_final(ctx, dest);
 }
 
 /* Interface routines for SHA-224 */
 void
 pg_sha224_init(pg_sha224_ctx *ctx)
 {
-	SHA224_Init((SHA256_CTX *) ctx);
+	digest_init(ctx, EVP_sha224());
 }
 
 void
 pg_sha224_update(pg_sha224_ctx *ctx, const uint8 *data, size_t len)
 {
-	SHA224_Update((SHA256_CTX *) ctx, data, len);
+	digest_update(ctx, data, len);
 }
 
 void
 pg_sha224_final(pg_sha224_ctx *ctx, uint8 *dest)
 {
-	SHA224_Final(dest, (SHA256_CTX *) ctx);
+	digest_final(ctx, dest);
 }

src/include/common/sha2.h

@@ -51,7 +51,7 @@
 #define _PG_SHA2_H_
 
 #ifdef USE_OPENSSL
-#include <openssl/sha.h>
+#include <openssl/evp.h>
 #endif
 
 /*** SHA224/256/384/512 Various Length Definitions ***********************/
@@ -70,10 +70,10 @@
 
 /* Context Structures for SHA224/256/384/512 */
 #ifdef USE_OPENSSL
-typedef SHA256_CTX pg_sha256_ctx;
-typedef SHA512_CTX pg_sha512_ctx;
-typedef SHA256_CTX pg_sha224_ctx;
-typedef SHA512_CTX pg_sha384_ctx;
+typedef EVP_MD_CTX *pg_sha256_ctx;
+typedef EVP_MD_CTX *pg_sha512_ctx;
+typedef EVP_MD_CTX *pg_sha224_ctx;
+typedef EVP_MD_CTX *pg_sha384_ctx;
 #else
 typedef struct pg_sha256_ctx
 {