![Ryan Mulligan](/assets/img/avatar_default.png)
Uses the HTTPS url for cases where the existing URL has a permanent redirect. For each domain, at least one fixed derivation URL was downloaded to test the domain is properly serving downloads. Also fixes jbake source URL, which was broken.
1148 lines
38 KiB
Nix
1148 lines
38 KiB
Nix
{ config, lib, pkgs, ... }:
|
|
|
|
with lib;
|
|
|
|
let
|
|
cfg = config.services.kubernetes;
|
|
|
|
skipAttrs = attrs: map (filterAttrs (k: v: k != "enable"))
|
|
(filter (v: !(hasAttr "enable" v) || v.enable) attrs);
|
|
|
|
infraContainer = pkgs.dockerTools.buildImage {
|
|
name = "pause";
|
|
tag = "latest";
|
|
contents = cfg.package.pause;
|
|
config.Cmd = "/bin/pause";
|
|
};
|
|
|
|
mkKubeConfig = name: cfg: pkgs.writeText "${name}-kubeconfig" (builtins.toJSON {
|
|
apiVersion = "v1";
|
|
kind = "Config";
|
|
clusters = [{
|
|
name = "local";
|
|
cluster.certificate-authority = cfg.caFile;
|
|
cluster.server = cfg.server;
|
|
}];
|
|
users = [{
|
|
name = "kubelet";
|
|
user = {
|
|
client-certificate = cfg.certFile;
|
|
client-key = cfg.keyFile;
|
|
};
|
|
}];
|
|
contexts = [{
|
|
context = {
|
|
cluster = "local";
|
|
user = "kubelet";
|
|
};
|
|
current-context = "kubelet-context";
|
|
}];
|
|
});
|
|
|
|
mkKubeConfigOptions = prefix: {
|
|
server = mkOption {
|
|
description = "${prefix} kube-apiserver server address.";
|
|
default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}";
|
|
type = types.str;
|
|
};
|
|
|
|
caFile = mkOption {
|
|
description = "${prefix} certificate authrority file used to connect to kube-apiserver.";
|
|
type = types.nullOr types.path;
|
|
default = cfg.caFile;
|
|
};
|
|
|
|
certFile = mkOption {
|
|
description = "${prefix} client certificate file used to connect to kube-apiserver.";
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
};
|
|
|
|
keyFile = mkOption {
|
|
description = "${prefix} client key file used to connect to kube-apiserver.";
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
};
|
|
};
|
|
|
|
kubeConfigDefaults = {
|
|
server = mkDefault cfg.kubeconfig.server;
|
|
caFile = mkDefault cfg.kubeconfig.caFile;
|
|
certFile = mkDefault cfg.kubeconfig.certFile;
|
|
keyFile = mkDefault cfg.kubeconfig.keyFile;
|
|
};
|
|
|
|
cniConfig = pkgs.buildEnv {
|
|
name = "kubernetes-cni-config";
|
|
paths = imap (i: entry:
|
|
pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry)
|
|
) cfg.kubelet.cni.config;
|
|
};
|
|
|
|
manifests = pkgs.buildEnv {
|
|
name = "kubernetes-manifests";
|
|
paths = mapAttrsToList (name: manifest:
|
|
pkgs.writeTextDir "${name}.json" (builtins.toJSON manifest)
|
|
) cfg.kubelet.manifests;
|
|
};
|
|
|
|
addons = pkgs.runCommand "kubernetes-addons" { } ''
|
|
mkdir -p $out
|
|
# since we are mounting the addons to the addon manager, they need to be copied
|
|
${concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (mapAttrsToList (name: addon:
|
|
pkgs.writeTextDir "${name}.json" (builtins.toJSON addon)
|
|
) (cfg.addonManager.addons))}
|
|
'';
|
|
|
|
taintOptions = { name, ... }: {
|
|
options = {
|
|
key = mkOption {
|
|
description = "Key of taint.";
|
|
default = name;
|
|
type = types.str;
|
|
};
|
|
value = mkOption {
|
|
description = "Value of taint.";
|
|
type = types.str;
|
|
};
|
|
effect = mkOption {
|
|
description = "Effect of taint.";
|
|
example = "NoSchedule";
|
|
type = types.enum ["NoSchedule" "PreferNoSchedule" "NoExecute"];
|
|
};
|
|
};
|
|
};
|
|
|
|
taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (mapAttrsToList (n: v: v) cfg.kubelet.taints);
|
|
|
|
# needed for flannel to pass options to docker
|
|
mkDockerOpts = pkgs.runCommand "mk-docker-opts" {
|
|
buildInputs = [ pkgs.makeWrapper ];
|
|
} ''
|
|
mkdir -p $out
|
|
cp ${pkgs.kubernetes.src}/cluster/centos/node/bin/mk-docker-opts.sh $out/mk-docker-opts.sh
|
|
|
|
# bashInteractive needed for `compgen`
|
|
makeWrapper ${pkgs.bashInteractive}/bin/bash $out/mk-docker-opts --add-flags "$out/mk-docker-opts.sh"
|
|
'';
|
|
in {
|
|
|
|
###### interface
|
|
|
|
options.services.kubernetes = {
|
|
roles = mkOption {
|
|
description = ''
|
|
Kubernetes role that this machine should take.
|
|
|
|
Master role will enable etcd, apiserver, scheduler and controller manager
|
|
services. Node role will enable etcd, docker, kubelet and proxy services.
|
|
'';
|
|
default = [];
|
|
type = types.listOf (types.enum ["master" "node"]);
|
|
};
|
|
|
|
package = mkOption {
|
|
description = "Kubernetes package to use.";
|
|
type = types.package;
|
|
default = pkgs.kubernetes;
|
|
defaultText = "pkgs.kubernetes";
|
|
};
|
|
|
|
verbose = mkOption {
|
|
description = "Kubernetes enable verbose mode for debugging.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
etcd = {
|
|
servers = mkOption {
|
|
description = "List of etcd servers. By default etcd is started, except if this option is changed.";
|
|
default = ["http://127.0.0.1:2379"];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
keyFile = mkOption {
|
|
description = "Etcd key file.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
certFile = mkOption {
|
|
description = "Etcd cert file.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
caFile = mkOption {
|
|
description = "Etcd ca file.";
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Default kubeconfig";
|
|
|
|
caFile = mkOption {
|
|
description = "Default kubernetes certificate authority";
|
|
type = types.nullOr types.path;
|
|
default = null;
|
|
};
|
|
|
|
dataDir = mkOption {
|
|
description = "Kubernetes root directory for managing kubelet files.";
|
|
default = "/var/lib/kubernetes";
|
|
type = types.path;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = [];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
apiserver = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes apiserver.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes apiserver listening address.";
|
|
default = "127.0.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
publicAddress = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver public listening address used for read only and
|
|
secure port.
|
|
'';
|
|
default = cfg.apiserver.address;
|
|
type = types.str;
|
|
};
|
|
|
|
advertiseAddress = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver IP address on which to advertise the apiserver
|
|
to members of the cluster. This address must be reachable by the rest
|
|
of the cluster.
|
|
'';
|
|
default = null;
|
|
type = types.nullOr types.str;
|
|
};
|
|
|
|
storageBackend = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver storage backend.
|
|
'';
|
|
default = "etcd3";
|
|
type = types.enum ["etcd2" "etcd3"];
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes apiserver listening port.";
|
|
default = 8080;
|
|
type = types.int;
|
|
};
|
|
|
|
securePort = mkOption {
|
|
description = "Kubernetes apiserver secure port.";
|
|
default = 443;
|
|
type = types.int;
|
|
};
|
|
|
|
tlsCertFile = mkOption {
|
|
description = "Kubernetes apiserver certificate file.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
tlsKeyFile = mkOption {
|
|
description = "Kubernetes apiserver private key file.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
clientCaFile = mkOption {
|
|
description = "Kubernetes apiserver CA file for client auth.";
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
tokenAuthFile = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver token authentication file. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/authentication.html"/>
|
|
'';
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
basicAuthFile = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver basic authentication file. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/authentication.html"/>
|
|
'';
|
|
default = pkgs.writeText "users" ''
|
|
kubernetes,admin,0
|
|
'';
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
authorizationMode = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/RBAC). See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/authorization.html"/>
|
|
'';
|
|
default = ["RBAC" "Node"];
|
|
type = types.listOf (types.enum ["AlwaysAllow" "AlwaysDeny" "ABAC" "RBAC" "Node"]);
|
|
};
|
|
|
|
authorizationPolicy = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver authorization policy file. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/authorization.html"/>
|
|
'';
|
|
default = [];
|
|
type = types.listOf types.attrs;
|
|
};
|
|
|
|
allowPrivileged = mkOption {
|
|
description = "Whether to allow privileged containers on Kubernetes.";
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
serviceClusterIpRange = mkOption {
|
|
description = ''
|
|
A CIDR notation IP range from which to assign service cluster IPs.
|
|
This must not overlap with any IP ranges assigned to nodes for pods.
|
|
'';
|
|
default = "10.0.0.0/24";
|
|
type = types.str;
|
|
};
|
|
|
|
runtimeConfig = mkOption {
|
|
description = ''
|
|
Api runtime configuration. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/cluster-management.html"/>
|
|
'';
|
|
default = "authentication.k8s.io/v1beta1=true";
|
|
example = "api/all=false,api/v1=true";
|
|
type = types.str;
|
|
};
|
|
|
|
admissionControl = mkOption {
|
|
description = ''
|
|
Kubernetes admission control plugins to use. See
|
|
<link xlink:href="https://kubernetes.io/docs/admin/admission-controllers/"/>
|
|
'';
|
|
default = ["NamespaceLifecycle" "LimitRanger" "ServiceAccount" "ResourceQuota" "DefaultStorageClass" "DefaultTolerationSeconds" "NodeRestriction"];
|
|
example = [
|
|
"NamespaceLifecycle" "NamespaceExists" "LimitRanger"
|
|
"SecurityContextDeny" "ServiceAccount" "ResourceQuota"
|
|
"PodSecurityPolicy" "NodeRestriction" "DefaultStorageClass"
|
|
];
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
serviceAccountKeyFile = mkOption {
|
|
description = ''
|
|
Kubernetes apiserver PEM-encoded x509 RSA private or public key file,
|
|
used to verify ServiceAccount tokens. By default tls private key file
|
|
is used.
|
|
'';
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeletClientCaFile = mkOption {
|
|
description = "Path to a cert file for connecting to kubelet.";
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeletClientCertFile = mkOption {
|
|
description = "Client certificate to use for connections to kubelet.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeletClientKeyFile = mkOption {
|
|
description = "Key to use for connections to kubelet.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeletHttps = mkOption {
|
|
description = "Whether to use https for connections to kubelet.";
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes apiserver extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
scheduler = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes scheduler.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes scheduler listening address.";
|
|
default = "127.0.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes scheduler listening port.";
|
|
default = 10251;
|
|
type = types.int;
|
|
};
|
|
|
|
leaderElect = mkOption {
|
|
description = "Whether to start leader election before executing main loop.";
|
|
type = types.bool;
|
|
default = true;
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Kubernetes scheduler";
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes scheduler extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
controllerManager = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes controller manager.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes controller manager listening address.";
|
|
default = "127.0.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes controller manager listening port.";
|
|
default = 10252;
|
|
type = types.int;
|
|
};
|
|
|
|
leaderElect = mkOption {
|
|
description = "Whether to start leader election before executing main loop.";
|
|
type = types.bool;
|
|
default = true;
|
|
};
|
|
|
|
serviceAccountKeyFile = mkOption {
|
|
description = ''
|
|
Kubernetes controller manager PEM-encoded private RSA key file used to
|
|
sign service account tokens
|
|
'';
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
rootCaFile = mkOption {
|
|
description = ''
|
|
Kubernetes controller manager certificate authority file included in
|
|
service account's token secret.
|
|
'';
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Kubernetes controller manager";
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes controller manager extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
kubelet = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes kubelet.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
seedDockerImages = mkOption {
|
|
description = "List of docker images to preload on system";
|
|
default = [];
|
|
type = types.listOf types.package;
|
|
};
|
|
|
|
registerNode = mkOption {
|
|
description = "Whether to auto register kubelet with API server.";
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes kubelet info server listening address.";
|
|
default = "0.0.0.0";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes kubelet info server listening port.";
|
|
default = 10250;
|
|
type = types.int;
|
|
};
|
|
|
|
tlsCertFile = mkOption {
|
|
description = "File containing x509 Certificate for HTTPS.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
tlsKeyFile = mkOption {
|
|
description = "File containing x509 private key matching tlsCertFile.";
|
|
default = null;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
clientCaFile = mkOption {
|
|
description = "Kubernetes apiserver CA file for client authentication.";
|
|
default = cfg.caFile;
|
|
type = types.nullOr types.path;
|
|
};
|
|
|
|
healthz = {
|
|
bind = mkOption {
|
|
description = "Kubernetes kubelet healthz listening address.";
|
|
default = "127.0.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
port = mkOption {
|
|
description = "Kubernetes kubelet healthz port.";
|
|
default = 10248;
|
|
type = types.int;
|
|
};
|
|
};
|
|
|
|
hostname = mkOption {
|
|
description = "Kubernetes kubelet hostname override.";
|
|
default = config.networking.hostName;
|
|
type = types.str;
|
|
};
|
|
|
|
allowPrivileged = mkOption {
|
|
description = "Whether to allow Kubernetes containers to request privileged mode.";
|
|
default = true;
|
|
type = types.bool;
|
|
};
|
|
|
|
cadvisorPort = mkOption {
|
|
description = "Kubernetes kubelet local cadvisor port.";
|
|
default = 4194;
|
|
type = types.int;
|
|
};
|
|
|
|
clusterDns = mkOption {
|
|
description = "Use alternative DNS.";
|
|
default = "10.1.0.1";
|
|
type = types.str;
|
|
};
|
|
|
|
clusterDomain = mkOption {
|
|
description = "Use alternative domain.";
|
|
default = config.services.kubernetes.addons.dns.clusterDomain;
|
|
type = types.str;
|
|
};
|
|
|
|
networkPlugin = mkOption {
|
|
description = "Network plugin to use by Kubernetes.";
|
|
type = types.nullOr (types.enum ["cni" "kubenet"]);
|
|
default = "kubenet";
|
|
};
|
|
|
|
cni = {
|
|
packages = mkOption {
|
|
description = "List of network plugin packages to install.";
|
|
type = types.listOf types.package;
|
|
default = [];
|
|
};
|
|
|
|
config = mkOption {
|
|
description = "Kubernetes CNI configuration.";
|
|
type = types.listOf types.attrs;
|
|
default = [];
|
|
example = literalExample ''
|
|
[{
|
|
"cniVersion": "0.2.0",
|
|
"name": "mynet",
|
|
"type": "bridge",
|
|
"bridge": "cni0",
|
|
"isGateway": true,
|
|
"ipMasq": true,
|
|
"ipam": {
|
|
"type": "host-local",
|
|
"subnet": "10.22.0.0/16",
|
|
"routes": [
|
|
{ "dst": "0.0.0.0/0" }
|
|
]
|
|
}
|
|
} {
|
|
"cniVersion": "0.2.0",
|
|
"type": "loopback"
|
|
}]
|
|
'';
|
|
};
|
|
};
|
|
|
|
manifests = mkOption {
|
|
description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
|
|
type = types.attrsOf types.attrs;
|
|
default = {};
|
|
};
|
|
|
|
applyManifests = mkOption {
|
|
description = "Whether to apply manifests (this is true for master node).";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
unschedulable = mkOption {
|
|
description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
taints = mkOption {
|
|
description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
|
|
default = {};
|
|
type = types.attrsOf (types.submodule [ taintOptions ]);
|
|
};
|
|
|
|
nodeIp = mkOption {
|
|
description = "IP address of the node. If set, kubelet will use this IP address for the node.";
|
|
default = null;
|
|
type = types.nullOr types.str;
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Kubelet";
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes kubelet extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
proxy = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes proxy.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
featureGates = mkOption {
|
|
description = "List set of feature gates";
|
|
default = cfg.featureGates;
|
|
type = types.listOf types.str;
|
|
};
|
|
|
|
address = mkOption {
|
|
description = "Kubernetes proxy listening address.";
|
|
default = "0.0.0.0";
|
|
type = types.str;
|
|
};
|
|
|
|
kubeconfig = mkKubeConfigOptions "Kubernetes proxy";
|
|
|
|
extraOpts = mkOption {
|
|
description = "Kubernetes proxy extra command line options.";
|
|
default = "";
|
|
type = types.str;
|
|
};
|
|
};
|
|
|
|
addonManager = {
|
|
enable = mkOption {
|
|
description = "Whether to enable Kubernetes addon manager.";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
addons = mkOption {
|
|
description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
|
|
default = { };
|
|
type = types.attrsOf (types.either types.attrs (types.listOf types.attrs));
|
|
example = literalExample ''
|
|
{
|
|
"my-service" = {
|
|
"apiVersion" = "v1";
|
|
"kind" = "Service";
|
|
"metadata" = {
|
|
"name" = "my-service";
|
|
"namespace" = "default";
|
|
};
|
|
"spec" = { ... };
|
|
};
|
|
}
|
|
// import <nixpkgs/nixos/modules/services/cluster/kubernetes/dashboard.nix> { cfg = config.services.kubernetes; };
|
|
'';
|
|
};
|
|
};
|
|
|
|
path = mkOption {
|
|
description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
|
|
type = types.listOf types.package;
|
|
default = [];
|
|
};
|
|
|
|
clusterCidr = mkOption {
|
|
description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
|
|
default = "10.1.0.0/16";
|
|
type = types.str;
|
|
};
|
|
|
|
flannel.enable = mkOption {
|
|
description = "Whether to enable flannel networking";
|
|
default = false;
|
|
type = types.bool;
|
|
};
|
|
|
|
};
|
|
|
|
###### implementation
|
|
|
|
config = mkMerge [
|
|
(mkIf cfg.kubelet.enable {
|
|
services.kubernetes.kubelet.seedDockerImages = [infraContainer];
|
|
|
|
systemd.services.kubelet-bootstrap = {
|
|
description = "Boostrap Kubelet";
|
|
wantedBy = ["kubernetes.target"];
|
|
after = ["docker.service" "network.target"];
|
|
path = with pkgs; [ docker ];
|
|
script = ''
|
|
${concatMapStrings (img: ''
|
|
echo "Seeding docker image: ${img}"
|
|
docker load <${img}
|
|
'') cfg.kubelet.seedDockerImages}
|
|
|
|
rm /opt/cni/bin/* || true
|
|
${concatMapStrings (package: ''
|
|
echo "Linking cni package: ${package}"
|
|
ln -fs ${package.plugins}/* /opt/cni/bin
|
|
'') cfg.kubelet.cni.packages}
|
|
'';
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
Type = "oneshot";
|
|
};
|
|
};
|
|
|
|
systemd.services.kubelet = {
|
|
description = "Kubernetes Kubelet Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "network.target" "docker.service" "kube-apiserver.service" "kubelet-bootstrap.service" ];
|
|
path = with pkgs; [ gitMinimal openssh docker utillinux iproute ethtool thin-provisioning-tools iptables socat ] ++ cfg.path;
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kubelet \
|
|
${optionalString cfg.kubelet.applyManifests
|
|
"--pod-manifest-path=${manifests}"} \
|
|
${optionalString (taints != "")
|
|
"--register-with-taints=${taints}"} \
|
|
--kubeconfig=${mkKubeConfig "kubelet" cfg.kubelet.kubeconfig} \
|
|
--require-kubeconfig \
|
|
--address=${cfg.kubelet.address} \
|
|
--port=${toString cfg.kubelet.port} \
|
|
--register-node=${boolToString cfg.kubelet.registerNode} \
|
|
${optionalString (cfg.kubelet.tlsCertFile != null)
|
|
"--tls-cert-file=${cfg.kubelet.tlsCertFile}"} \
|
|
${optionalString (cfg.kubelet.tlsKeyFile != null)
|
|
"--tls-private-key-file=${cfg.kubelet.tlsKeyFile}"} \
|
|
${optionalString (cfg.kubelet.clientCaFile != null)
|
|
"--client-ca-file=${cfg.kubelet.clientCaFile}"} \
|
|
--authentication-token-webhook \
|
|
--authentication-token-webhook-cache-ttl="10s" \
|
|
--authorization-mode=Webhook \
|
|
--healthz-bind-address=${cfg.kubelet.healthz.bind} \
|
|
--healthz-port=${toString cfg.kubelet.healthz.port} \
|
|
--hostname-override=${cfg.kubelet.hostname} \
|
|
--allow-privileged=${boolToString cfg.kubelet.allowPrivileged} \
|
|
--root-dir=${cfg.dataDir} \
|
|
--cadvisor_port=${toString cfg.kubelet.cadvisorPort} \
|
|
${optionalString (cfg.kubelet.clusterDns != "")
|
|
"--cluster-dns=${cfg.kubelet.clusterDns}"} \
|
|
${optionalString (cfg.kubelet.clusterDomain != "")
|
|
"--cluster-domain=${cfg.kubelet.clusterDomain}"} \
|
|
--pod-infra-container-image=pause \
|
|
${optionalString (cfg.kubelet.networkPlugin != null)
|
|
"--network-plugin=${cfg.kubelet.networkPlugin}"} \
|
|
--cni-conf-dir=${cniConfig} \
|
|
--hairpin-mode=hairpin-veth \
|
|
${optionalString (cfg.kubelet.nodeIp != null)
|
|
"--node-ip=${cfg.kubelet.nodeIp}"} \
|
|
${optionalString (cfg.kubelet.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \
|
|
${optionalString cfg.verbose "--v=6 --log_flush_frequency=1s"} \
|
|
${cfg.kubelet.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
};
|
|
};
|
|
|
|
# Allways include cni plugins
|
|
services.kubernetes.kubelet.cni.packages = [pkgs.cni];
|
|
|
|
boot.kernelModules = ["br_netfilter"];
|
|
|
|
services.kubernetes.kubelet.kubeconfig = kubeConfigDefaults;
|
|
})
|
|
|
|
(mkIf (cfg.kubelet.applyManifests && cfg.kubelet.enable) {
|
|
environment.etc = mapAttrs' (name: manifest:
|
|
nameValuePair "kubernetes/manifests/${name}.json" {
|
|
text = builtins.toJSON manifest;
|
|
mode = "0755";
|
|
}
|
|
) cfg.kubelet.manifests;
|
|
})
|
|
|
|
(mkIf (cfg.kubelet.unschedulable && cfg.kubelet.enable) {
|
|
services.kubernetes.kubelet.taints.unschedulable = {
|
|
value = "true";
|
|
effect = "NoSchedule";
|
|
};
|
|
})
|
|
|
|
(mkIf cfg.apiserver.enable {
|
|
systemd.services.kube-apiserver = {
|
|
description = "Kubernetes Kubelet Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "network.target" "docker.service" ];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kube-apiserver \
|
|
--etcd-servers=${concatStringsSep "," cfg.etcd.servers} \
|
|
${optionalString (cfg.etcd.caFile != null)
|
|
"--etcd-cafile=${cfg.etcd.caFile}"} \
|
|
${optionalString (cfg.etcd.certFile != null)
|
|
"--etcd-certfile=${cfg.etcd.certFile}"} \
|
|
${optionalString (cfg.etcd.keyFile != null)
|
|
"--etcd-keyfile=${cfg.etcd.keyFile}"} \
|
|
--insecure-port=${toString cfg.apiserver.port} \
|
|
--bind-address=0.0.0.0 \
|
|
${optionalString (cfg.apiserver.advertiseAddress != null)
|
|
"--advertise-address=${cfg.apiserver.advertiseAddress}"} \
|
|
--allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\
|
|
${optionalString (cfg.apiserver.tlsCertFile != null)
|
|
"--tls-cert-file=${cfg.apiserver.tlsCertFile}"} \
|
|
${optionalString (cfg.apiserver.tlsKeyFile != null)
|
|
"--tls-private-key-file=${cfg.apiserver.tlsKeyFile}"} \
|
|
${optionalString (cfg.apiserver.tokenAuthFile != null)
|
|
"--token-auth-file=${cfg.apiserver.tokenAuthFile}"} \
|
|
${optionalString (cfg.apiserver.basicAuthFile != null)
|
|
"--basic-auth-file=${cfg.apiserver.basicAuthFile}"} \
|
|
--kubelet-https=${if cfg.apiserver.kubeletHttps then "true" else "false"} \
|
|
${optionalString (cfg.apiserver.kubeletClientCaFile != null)
|
|
"--kubelet-certificate-authority=${cfg.apiserver.kubeletClientCaFile}"} \
|
|
${optionalString (cfg.apiserver.kubeletClientCertFile != null)
|
|
"--kubelet-client-certificate=${cfg.apiserver.kubeletClientCertFile}"} \
|
|
${optionalString (cfg.apiserver.kubeletClientKeyFile != null)
|
|
"--kubelet-client-key=${cfg.apiserver.kubeletClientKeyFile}"} \
|
|
${optionalString (cfg.apiserver.clientCaFile != null)
|
|
"--client-ca-file=${cfg.apiserver.clientCaFile}"} \
|
|
--authorization-mode=${concatStringsSep "," cfg.apiserver.authorizationMode} \
|
|
${optionalString (elem "ABAC" cfg.apiserver.authorizationMode)
|
|
"--authorization-policy-file=${
|
|
pkgs.writeText "kube-auth-policy.jsonl"
|
|
(concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.apiserver.authorizationPolicy)
|
|
}"
|
|
} \
|
|
--secure-port=${toString cfg.apiserver.securePort} \
|
|
--service-cluster-ip-range=${cfg.apiserver.serviceClusterIpRange} \
|
|
${optionalString (cfg.apiserver.runtimeConfig != "")
|
|
"--runtime-config=${cfg.apiserver.runtimeConfig}"} \
|
|
--admission_control=${concatStringsSep "," cfg.apiserver.admissionControl} \
|
|
${optionalString (cfg.apiserver.serviceAccountKeyFile!=null)
|
|
"--service-account-key-file=${cfg.apiserver.serviceAccountKeyFile}"} \
|
|
${optionalString cfg.verbose "--v=6"} \
|
|
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
|
|
--storage-backend=${cfg.apiserver.storageBackend} \
|
|
${optionalString (cfg.kubelet.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.kubelet.featureGates}"} \
|
|
${cfg.apiserver.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
AmbientCapabilities = "cap_net_bind_service";
|
|
Restart = "on-failure";
|
|
RestartSec = 5;
|
|
};
|
|
};
|
|
})
|
|
|
|
(mkIf cfg.scheduler.enable {
|
|
systemd.services.kube-scheduler = {
|
|
description = "Kubernetes Scheduler Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kube-scheduler \
|
|
--address=${cfg.scheduler.address} \
|
|
--port=${toString cfg.scheduler.port} \
|
|
--leader-elect=${boolToString cfg.scheduler.leaderElect} \
|
|
--kubeconfig=${mkKubeConfig "kube-scheduler" cfg.scheduler.kubeconfig} \
|
|
${optionalString cfg.verbose "--v=6"} \
|
|
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
|
|
${optionalString (cfg.scheduler.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.scheduler.featureGates}"} \
|
|
${cfg.scheduler.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
};
|
|
};
|
|
|
|
services.kubernetes.scheduler.kubeconfig = kubeConfigDefaults;
|
|
})
|
|
|
|
(mkIf cfg.controllerManager.enable {
|
|
systemd.services.kube-controller-manager = {
|
|
description = "Kubernetes Controller Manager Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
serviceConfig = {
|
|
RestartSec = "30s";
|
|
Restart = "on-failure";
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kube-controller-manager \
|
|
--address=${cfg.controllerManager.address} \
|
|
--port=${toString cfg.controllerManager.port} \
|
|
--kubeconfig=${mkKubeConfig "kube-controller-manager" cfg.controllerManager.kubeconfig} \
|
|
--leader-elect=${boolToString cfg.controllerManager.leaderElect} \
|
|
${if (cfg.controllerManager.serviceAccountKeyFile!=null)
|
|
then "--service-account-private-key-file=${cfg.controllerManager.serviceAccountKeyFile}"
|
|
else "--service-account-private-key-file=/var/run/kubernetes/apiserver.key"} \
|
|
${if (cfg.controllerManager.rootCaFile!=null)
|
|
then "--root-ca-file=${cfg.controllerManager.rootCaFile}"
|
|
else "--root-ca-file=/var/run/kubernetes/apiserver.crt"} \
|
|
${optionalString (cfg.clusterCidr!=null)
|
|
"--cluster-cidr=${cfg.clusterCidr}"} \
|
|
--allocate-node-cidrs=true \
|
|
${optionalString (cfg.controllerManager.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.controllerManager.featureGates}"} \
|
|
${optionalString cfg.verbose "--v=6"} \
|
|
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
|
|
${cfg.controllerManager.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
};
|
|
path = cfg.path;
|
|
};
|
|
|
|
services.kubernetes.controllerManager.kubeconfig = kubeConfigDefaults;
|
|
})
|
|
|
|
(mkIf cfg.proxy.enable {
|
|
systemd.services.kube-proxy = {
|
|
description = "Kubernetes Proxy Service";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
path = [pkgs.iptables pkgs.conntrack_tools];
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = ''${cfg.package}/bin/kube-proxy \
|
|
--kubeconfig=${mkKubeConfig "kube-proxy" cfg.proxy.kubeconfig} \
|
|
--bind-address=${cfg.proxy.address} \
|
|
${optionalString (cfg.proxy.featureGates != [])
|
|
"--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.proxy.featureGates}"} \
|
|
${optionalString cfg.verbose "--v=6"} \
|
|
${optionalString cfg.verbose "--log-flush-frequency=1s"} \
|
|
${optionalString (cfg.clusterCidr!=null)
|
|
"--cluster-cidr=${cfg.clusterCidr}"} \
|
|
${cfg.proxy.extraOpts}
|
|
'';
|
|
WorkingDirectory = cfg.dataDir;
|
|
};
|
|
};
|
|
|
|
# kube-proxy needs iptables
|
|
networking.firewall.enable = mkDefault true;
|
|
|
|
services.kubernetes.proxy.kubeconfig = kubeConfigDefaults;
|
|
})
|
|
|
|
(mkIf (any (el: el == "master") cfg.roles) {
|
|
virtualisation.docker.enable = mkDefault true;
|
|
services.kubernetes.kubelet.enable = mkDefault true;
|
|
services.kubernetes.kubelet.allowPrivileged = mkDefault true;
|
|
services.kubernetes.kubelet.applyManifests = mkDefault true;
|
|
services.kubernetes.apiserver.enable = mkDefault true;
|
|
services.kubernetes.scheduler.enable = mkDefault true;
|
|
services.kubernetes.controllerManager.enable = mkDefault true;
|
|
services.etcd.enable = mkDefault (cfg.etcd.servers == ["http://127.0.0.1:2379"]);
|
|
services.kubernetes.addonManager.enable = mkDefault true;
|
|
services.kubernetes.proxy.enable = mkDefault true;
|
|
})
|
|
|
|
# if this node is only a master make it unschedulable by default
|
|
(mkIf (all (el: el == "master") cfg.roles) {
|
|
services.kubernetes.kubelet.unschedulable = mkDefault true;
|
|
})
|
|
|
|
(mkIf (any (el: el == "node") cfg.roles) {
|
|
virtualisation.docker = {
|
|
enable = mkDefault true;
|
|
|
|
# kubernetes needs access to logs
|
|
logDriver = mkDefault "json-file";
|
|
|
|
# iptables must be disabled for kubernetes
|
|
extraOptions = "--iptables=false --ip-masq=false";
|
|
};
|
|
|
|
services.kubernetes.kubelet.enable = mkDefault true;
|
|
services.kubernetes.proxy.enable = mkDefault true;
|
|
})
|
|
|
|
(mkIf cfg.addonManager.enable {
|
|
environment.etc."kubernetes/addons".source = "${addons}/";
|
|
|
|
systemd.services.kube-addon-manager = {
|
|
description = "Kubernetes addon manager";
|
|
wantedBy = [ "kubernetes.target" ];
|
|
after = [ "kube-apiserver.service" ];
|
|
environment.ADDON_PATH = "/etc/kubernetes/addons/";
|
|
serviceConfig = {
|
|
Slice = "kubernetes.slice";
|
|
ExecStart = "${cfg.package}/bin/kube-addons";
|
|
WorkingDirectory = cfg.dataDir;
|
|
User = "kubernetes";
|
|
Group = "kubernetes";
|
|
};
|
|
};
|
|
})
|
|
|
|
(mkIf (
|
|
cfg.apiserver.enable ||
|
|
cfg.scheduler.enable ||
|
|
cfg.controllerManager.enable ||
|
|
cfg.kubelet.enable ||
|
|
cfg.proxy.enable
|
|
) {
|
|
systemd.targets.kubernetes = {
|
|
description = "Kubernetes";
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d /opt/cni/bin 0755 root root -"
|
|
"d /var/run/kubernetes 0755 kubernetes kubernetes -"
|
|
"d /var/lib/kubernetes 0755 kubernetes kubernetes -"
|
|
];
|
|
|
|
environment.systemPackages = [ cfg.package ];
|
|
users.extraUsers = singleton {
|
|
name = "kubernetes";
|
|
uid = config.ids.uids.kubernetes;
|
|
description = "Kubernetes user";
|
|
extraGroups = [ "docker" ];
|
|
group = "kubernetes";
|
|
home = cfg.dataDir;
|
|
createHome = true;
|
|
};
|
|
users.extraGroups.kubernetes.gid = config.ids.gids.kubernetes;
|
|
|
|
# dns addon is enabled by default
|
|
services.kubernetes.addons.dns.enable = mkDefault true;
|
|
})
|
|
|
|
(mkIf cfg.flannel.enable {
|
|
services.flannel = {
|
|
enable = mkDefault true;
|
|
network = mkDefault cfg.clusterCidr;
|
|
etcd = mkDefault {
|
|
endpoints = cfg.etcd.servers;
|
|
inherit (cfg.etcd) caFile certFile keyFile;
|
|
};
|
|
};
|
|
|
|
services.kubernetes.kubelet = {
|
|
networkPlugin = mkDefault "cni";
|
|
cni.config = mkDefault [{
|
|
name = "mynet";
|
|
type = "flannel";
|
|
delegate = {
|
|
isDefaultGateway = true;
|
|
bridge = "docker0";
|
|
};
|
|
}];
|
|
};
|
|
|
|
systemd.services."mk-docker-opts" = {
|
|
description = "Pre-Docker Actions";
|
|
wantedBy = [ "flannel.service" ];
|
|
before = [ "docker.service" ];
|
|
after = [ "flannel.service" ];
|
|
path = [ pkgs.gawk pkgs.gnugrep ];
|
|
script = ''
|
|
mkdir -p /run/flannel
|
|
${mkDockerOpts}/mk-docker-opts -d /run/flannel/docker
|
|
'';
|
|
serviceConfig.Type = "oneshot";
|
|
};
|
|
systemd.services.docker.serviceConfig.EnvironmentFile = "/run/flannel/docker";
|
|
|
|
# read environment variables generated by mk-docker-opts
|
|
virtualisation.docker.extraOptions = "$DOCKER_OPTS";
|
|
|
|
networking.firewall.allowedUDPPorts = [
|
|
8285 # flannel udp
|
|
8472 # flannel vxlan
|
|
];
|
|
})
|
|
];
|
|
}
|