blob: 1107843b7215203236fec654b4ccc3a6e8cb30c0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
# Test a minimal hbase cluster
{ package, pkgs, ... }:
import ../make-test-python.nix (
{
hadoop ? package,
hbase ? pkgs.hbase,
...
}:
with pkgs.lib;
{
name = "hadoop-${package.version}-hbase";
nodes =
let
coreSite = {
"fs.defaultFS" = "hdfs://namenode:8020";
};
defOpts = {
enable = true;
openFirewall = true;
};
zookeeperQuorum = "zookeeper";
in
{
zookeeper =
{ ... }:
{
services.zookeeper.enable = true;
networking.firewall.allowedTCPPorts = [ 2181 ];
};
namenode =
{ ... }:
{
services.hadoop = {
hdfs = {
namenode = defOpts // {
formatOnInit = true;
};
};
inherit coreSite;
};
};
datanode =
{ ... }:
{
virtualisation.diskSize = 8192;
services.hadoop = {
hdfs.datanode = defOpts;
inherit coreSite;
};
};
master =
{ ... }:
{
services.hadoop = {
inherit coreSite;
hbase = {
inherit zookeeperQuorum;
master = defOpts // {
initHDFS = true;
};
};
};
};
regionserver =
{ ... }:
{
services.hadoop = {
inherit coreSite;
hbase = {
inherit zookeeperQuorum;
regionServer = defOpts;
};
};
};
thrift =
{ ... }:
{
services.hadoop = {
inherit coreSite;
hbase = {
inherit zookeeperQuorum;
thrift = defOpts;
};
};
};
rest =
{ ... }:
{
services.hadoop = {
inherit coreSite;
hbase = {
inherit zookeeperQuorum;
rest = defOpts;
};
};
};
};
testScript = ''
start_all()
# wait for HDFS cluster
namenode.wait_for_unit("hdfs-namenode")
namenode.wait_for_unit("network.target")
namenode.wait_for_open_port(8020)
namenode.wait_for_open_port(9870)
datanode.wait_for_unit("hdfs-datanode")
datanode.wait_for_unit("network.target")
datanode.wait_for_open_port(9864)
datanode.wait_for_open_port(9866)
datanode.wait_for_open_port(9867)
# wait for ZK
zookeeper.wait_for_unit("zookeeper")
zookeeper.wait_for_open_port(2181)
# wait for HDFS cluster to be RW
datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
# wait for HBase to start up
master.wait_for_unit("hbase-master")
regionserver.wait_for_unit("hbase-regionserver")
assert "1 active master, 0 backup masters, 1 servers" in master.succeed("echo status | HADOOP_USER_NAME=hbase hbase shell -n")
regionserver.wait_until_succeeds("echo \"create 't1','f1'\" | HADOOP_USER_NAME=hbase hbase shell -n")
assert "NAME => 'f1'" in regionserver.succeed("echo \"describe 't1'\" | HADOOP_USER_NAME=hbase hbase shell -n")
rest.wait_for_open_port(8080)
assert "${hbase.version}" in regionserver.succeed("curl http://rest:8080/version/cluster")
thrift.wait_for_open_port(9090)
'';
meta.maintainers = with maintainers; [ illustris ];
}
)
|
