nixpkgs/nixos/tests/kubernetes/base.nix
Alberto Berti 0c841f98d0 Kubernetes tests: increase the size of the VM from 700MB to 1.5GB
VMs were starving, many of the daemons were unable to complete their
tasks resulting in tests failures.

Turned off verbose output from k8s components as it consumes even more resources, and useful error messages actually drown in debug-clutter
2018-05-25 11:10:15 +02:00

113 lines
4.1 KiB
Nix

{ system ? builtins.currentSystem }:
with import ../../lib/testing.nix { inherit system; };
with pkgs.lib;
let
mkKubernetesBaseTest =
{ name, domain ? "my.zyx", test, machines
, pkgs ? import <nixpkgs> { inherit system; }
, certs ? import ./certs.nix { inherit pkgs; externalDomain = domain; kubelets = attrNames machines; }
, extraConfiguration ? null }:
let
masterName = head (filter (machineName: any (role: role == "master") machines.${machineName}.roles) (attrNames machines));
master = machines.${masterName};
extraHosts = ''
${master.ip} etcd.${domain}
${master.ip} api.${domain}
${concatMapStringsSep "\n" (machineName: "${machines.${machineName}.ip} ${machineName}.${domain}") (attrNames machines)}
'';
in makeTest {
inherit name;
nodes = mapAttrs (machineName: machine:
{ config, pkgs, lib, nodes, ... }:
mkMerge [
{
virtualisation.memorySize = mkDefault 1536;
virtualisation.diskSize = mkDefault 4096;
networking = {
inherit domain extraHosts;
primaryIPAddress = mkForce machine.ip;
firewall = {
allowedTCPPorts = [
10250 # kubelet
];
trustedInterfaces = ["docker0"];
extraCommands = concatMapStrings (node: ''
iptables -A INPUT -s ${node.config.networking.primaryIPAddress} -j ACCEPT
'') (attrValues nodes);
};
};
programs.bash.enableCompletion = true;
environment.variables = {
ETCDCTL_CERT_FILE = "${certs.worker}/etcd-client.pem";
ETCDCTL_KEY_FILE = "${certs.worker}/etcd-client-key.pem";
ETCDCTL_CA_FILE = "${certs.worker}/ca.pem";
ETCDCTL_PEERS = "https://etcd.${domain}:2379";
};
services.flannel.iface = "eth1";
services.kubernetes.apiserver.advertiseAddress = master.ip;
}
(optionalAttrs (any (role: role == "master") machine.roles) {
networking.firewall.allowedTCPPorts = [
2379 2380 # etcd
443 # kubernetes apiserver
];
services.etcd = {
enable = true;
certFile = "${certs.master}/etcd.pem";
keyFile = "${certs.master}/etcd-key.pem";
trustedCaFile = "${certs.master}/ca.pem";
peerClientCertAuth = true;
listenClientUrls = ["https://0.0.0.0:2379"];
listenPeerUrls = ["https://0.0.0.0:2380"];
advertiseClientUrls = ["https://etcd.${config.networking.domain}:2379"];
initialCluster = ["${masterName}=https://etcd.${config.networking.domain}:2380"];
initialAdvertisePeerUrls = ["https://etcd.${config.networking.domain}:2380"];
};
})
(import ./kubernetes-common.nix { inherit (machine) roles; inherit pkgs config certs; })
(optionalAttrs (machine ? "extraConfiguration") (machine.extraConfiguration { inherit config pkgs lib nodes; }))
(optionalAttrs (extraConfiguration != null) (extraConfiguration { inherit config pkgs lib nodes; }))
]
) machines;
testScript = ''
startAll;
${test}
'';
};
mkKubernetesMultiNodeTest = attrs: mkKubernetesBaseTest ({
machines = {
machine1 = {
roles = ["master"];
ip = "192.168.1.1";
};
machine2 = {
roles = ["node"];
ip = "192.168.1.2";
};
};
} // attrs // {
name = "kubernetes-${attrs.name}-multinode";
});
mkKubernetesSingleNodeTest = attrs: mkKubernetesBaseTest ({
machines = {
machine1 = {
roles = ["master" "node"];
ip = "192.168.1.1";
};
};
} // attrs // {
name = "kubernetes-${attrs.name}-singlenode";
});
in {
inherit mkKubernetesBaseTest mkKubernetesSingleNodeTest mkKubernetesMultiNodeTest;
}