aboutsummaryrefslogtreecommitdiff
path: root/ports/librtmp/dh.patch
blob: 4b03453547ed39c9f39907c75d31e3b67ddcd280 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
diff --git a/librtmp/dh.h b/librtmp/dh.h
index 8e285a60c..ea562d200 100644
--- a/librtmp/dh.h	
+++ b/librtmp/dh.h
@@ -139,11 +139,14 @@ typedef BIGNUM * MP_t;
 #define MP_setbin(u,buf,len)	BN_bn2bin(u,buf)
 #define MP_getbin(u,buf,len)	u = BN_bin2bn(buf,len,0)
 
+
 #define MDH	DH
 #define MDH_new()	DH_new()
 #define MDH_free(dh)	DH_free(dh)
 #define MDH_generate_key(dh)	DH_generate_key(dh)
 #define MDH_compute_key(secret, seclen, pub, dh)	DH_compute_key(secret, pub, dh)
+#define MPH_set_pqg(dh, p, q, g, res)  res = DH_set0_pqg(dh, p, q, g)
+#define MPH_set_length(dh, len, res)  res = DH_set_length(dh,len)
 
 #endif
 
@@ -152,7 +155,7 @@ typedef BIGNUM * MP_t;
 
 /* RFC 2631, Section 2.1.5, http://www.ietf.org/rfc/rfc2631.txt */
 static int
-isValidPublicKey(MP_t y, MP_t p, MP_t q)
+isValidPublicKey(const MP_t y,const MP_t p, MP_t q)
 {
   int ret = TRUE;
   MP_t bn;
@@ -211,20 +214,33 @@ DHInit(int nKeyBits)
   if (!dh)
     goto failed;
 
-  MP_new(dh->g);
+  MP_t g,p;
+  MP_new(g);
 
-  if (!dh->g)
+  if (!g) 
+  {
     goto failed;
+  }
 
-  MP_gethex(dh->p, P1024, res);	/* prime P1024, see dhgroups.h */
+  DH_get0_pqg(dh, (BIGNUM const**)&p, NULL, NULL);
+  MP_gethex(p, P1024, res);	/* prime P1024, see dhgroups.h */
   if (!res)
     {
       goto failed;
     }
 
-  MP_set_w(dh->g, 2);	/* base 2 */
-
-  dh->length = nKeyBits;
+  MP_set_w(g, 2);	/* base 2 */
+  MPH_set_pqg(dh,p,NULL,g, res);
+  if (!res)
+  {
+    MP_free(g);
+    goto failed;
+  }
+  MPH_set_length(dh,nKeyBits, res);
+  if (!res)
+  {
+    goto failed;
+  }
   return dh;
 
 failed:
@@ -250,14 +267,11 @@ DHGenerateKey(MDH *dh)
 
       MP_gethex(q1, Q1024, res);
       assert(res);
-
-      res = isValidPublicKey(dh->pub_key, dh->p, q1);
+      res = isValidPublicKey(DH_get0_pub_key(dh), DH_get0_p(dh), q1);
       if (!res)
-	{
-	  MP_free(dh->pub_key);
-	  MP_free(dh->priv_key);
-	  dh->pub_key = dh->priv_key = 0;
-	}
+        {
+              MDH_free(dh); // Cannot set priv_key to nullptr so there is no way to generate a new pub/priv key pair in openssl 1.1.1.
+        }
 
       MP_free(q1);
     }
@@ -272,15 +286,16 @@ static int
 DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
 {
   int len;
-  if (!dh || !dh->pub_key)
+  MP_t pub = DH_get0_pub_key(dh);
+  if (!dh || !pub)
     return 0;
 
-  len = MP_bytes(dh->pub_key);
+  len = MP_bytes(pub);
   if (len <= 0 || len > (int) nPubkeyLen)
     return 0;
 
   memset(pubkey, 0, nPubkeyLen);
-  MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len);
+  MP_setbin(pub, pubkey + (nPubkeyLen - len), len);
   return 1;
 }
 
@@ -288,15 +303,16 @@ DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
 static int
 DHGetPrivateKey(MDH *dh, uint8_t *privkey, size_t nPrivkeyLen)
 {
-  if (!dh || !dh->priv_key)
+  MP_t priv = DH_get0_priv_key(dh);
+  if (!dh || !priv)
     return 0;
 
-  int len = MP_bytes(dh->priv_key);
+  int len = MP_bytes(priv);
   if (len <= 0 || len > (int) nPrivkeyLen)
     return 0;
 
   memset(privkey, 0, nPrivkeyLen);
-  MP_setbin(dh->priv_key, privkey + (nPrivkeyLen - len), len);
+  MP_setbin(priv, privkey + (nPrivkeyLen - len), len);
   return 1;
 }
 #endif
@@ -322,7 +338,7 @@ DHComputeSharedSecretKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen,
   MP_gethex(q1, Q1024, len);
   assert(len);
 
-  if (isValidPublicKey(pubkeyBn, dh->p, q1))
+  if (isValidPublicKey(pubkeyBn, DH_get0_p(dh), q1))
     res = MDH_compute_key(secret, nPubkeyLen, pubkeyBn, dh);
   else
     res = -1;