1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
/*
Test suite for curl-impersonate
Abstract:
Uses the test suite from the curl-impersonate source repo which:
1. Performs requests with libcurl and captures the TLS client-hello
packets with tcpdump to compare against known-good signatures
2. Spins up an nghttpd2 server to test client HTTP/2 headers against
known-good headers
See https://github.com/lexiforest/curl-impersonate/tree/main/tests/signatures
for details.
Notes:
- We need to have our own web server running because the tests expect to be able
to hit domains like wikipedia.org and the sandbox has no internet
- We need to be able to do (verifying) TLS handshakes without internet access.
We do that by creating a trusted CA and issuing a cert that includes
all of the test domains as subject-alternative names and then spoofs the
hostnames in /etc/hosts.
- We started skipping the test_http2_headers test due to log format differences
between the nghttpd2 version in nixpkgs and the outdated one curl-impersonate
uses upstream for its tests.
- test_http3_fingerprint is skipped because it requires connecting to
third-party endpoint (fp.impersonate.pro) which isn't reachable from the test VM
*/
{ config, lib, ... }:
let
pkgs = config.node.pkgs;
# Update with domains in TestImpersonate.TEST_URLS if needed from:
# https://github.com/lexiforest/curl-impersonate/blob/main/tests/test_impersonate.py
domains = [
"www.wikimedia.org"
"www.wikipedia.org"
"www.mozilla.org"
"www.apache.org"
"git-scm.com"
];
tls-certs =
let
# Configure CA with X.509 v3 extensions that would be trusted by curl
ca-cert-conf = pkgs.writeText "curl-impersonate-ca.cnf" ''
basicConstraints = critical, CA:TRUE
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer:always
keyUsage = critical, cRLSign, digitalSignature, keyCertSign
'';
# Configure leaf certificate with X.509 v3 extensions that would be trusted
# by curl and set subject-alternative names for test domains
tls-cert-conf = pkgs.writeText "curl-impersonate-tls.cnf" ''
basicConstraints = critical, CA:FALSE
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer:always
keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment, keyAgreement
extendedKeyUsage = critical, serverAuth
subjectAltName = @alt_names
[alt_names]
${lib.concatStringsSep "\n" (lib.imap0 (idx: domain: "DNS.${toString idx} = ${domain}") domains)}
'';
in
pkgs.runCommand "curl-impersonate-test-certs"
{
nativeBuildInputs = [ pkgs.openssl ];
}
''
# create CA certificate and key
openssl req -newkey rsa:4096 -keyout ca-key.pem -out ca-csr.pem -nodes -subj '/CN=curl-impersonate-ca.nixos.test'
openssl x509 -req -sha512 -in ca-csr.pem -key ca-key.pem -out ca.pem -extfile ${ca-cert-conf} -days 36500
openssl x509 -in ca.pem -text
# create server certificate and key
openssl req -newkey rsa:4096 -keyout key.pem -out csr.pem -nodes -subj '/CN=curl-impersonate.nixos.test'
openssl x509 -req -sha512 -in csr.pem -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile ${tls-cert-conf} -days 36500
openssl x509 -in cert.pem -text
# output CA cert and server cert and key
mkdir -p $out
cp key.pem cert.pem ca.pem $out
'';
# Test script
curl-impersonate-test =
let
# Build miniature libcurl client used by test driver
minicurl =
pkgs.runCommandCC "minicurl"
{
buildInputs = [ pkgs.curl ];
}
''
mkdir -p $out/bin
sed -e 's/opts->local_port_end - opts->local_port_start);/(long)(opts->local_port_end - opts->local_port_start));/' \
-e 's/^\( *\)opts->local_port_start);/\1(long)opts->local_port_start);/' \
-e 's/CURLOPT_SSL_VERIFYPEER, 0)/CURLOPT_SSL_VERIFYPEER, 0L)/' \
-e 's/CURLOPT_SSL_VERIFYHOST, 0)/CURLOPT_SSL_VERIFYHOST, 0L)/' \
${pkgs.curl-impersonate.src}/tests/minicurl.c > minicurl.c
$CC -Wall -Werror -o $out/bin/minicurl minicurl.c `curl-config --libs`
'';
in
pkgs.writeShellScript "curl-impersonate-test" ''
set -euxo pipefail
# Test driver requirements
export PATH="${
with pkgs;
lib.makeBinPath [
bash
coreutils
python3Packages.pytest
nghttp2
tcpdump
]
}"
export PYTHONPATH="${
with pkgs.python3Packages;
makePythonPath [
pyyaml
pytest-asyncio
dpkt
ts1-signatures
]
}"
# Prepare test root prefix
mkdir -p usr/{bin,lib}
cp -rs ${pkgs.curl-impersonate}/* ${minicurl}/* usr/
cp -r ${pkgs.curl-impersonate.src}/tests ./
# Run tests
cd tests
pytest . --install-dir ../usr --capture-interface eth1 --exitfirst -k 'not test_http2_headers and not test_http3_fingerprint'
'';
in
{
name = "curl-impersonate";
meta = {
maintainers = with lib.maintainers; [
ui-1
];
};
nodes = {
web = { ... }: {
networking.firewall.allowedTCPPorts = [
80
443
];
services = {
nginx = {
enable = true;
virtualHosts."curl-impersonate.nixos.test" = {
default = true;
addSSL = true;
sslCertificate = "${tls-certs}/cert.pem";
sslCertificateKey = "${tls-certs}/key.pem";
};
};
};
};
curl =
{
nodes,
lib,
...
}:
{
networking.extraHosts = lib.concatStringsSep "\n" (
map (domain: "${nodes.web.networking.primaryIPAddress} ${domain}") domains
);
security.pki.certificateFiles = [ "${tls-certs}/ca.pem" ];
};
};
testScript = { ... }: ''
start_all()
with subtest("Wait for network"):
web.systemctl("start network-online.target")
curl.systemctl("start network-online.target")
web.wait_for_unit("network-online.target")
curl.wait_for_unit("network-online.target")
with subtest("Wait for web server"):
web.wait_for_unit("nginx.service")
web.wait_for_open_port(443)
with subtest("Run curl-impersonate tests"):
curl.succeed("${curl-impersonate-test}")
'';
}
|
