From ea6985ffc1af67fe11581c0e4c0a28e852f26c0b Mon Sep 17 00:00:00 2001 From: Christian Albrecht Date: Mon, 11 Mar 2019 10:53:59 +0100 Subject: [PATCH] Cleanup pki: flannel --- .../services/cluster/kubernetes/flannel.nix | 49 +++++++++++++++---- .../services/cluster/kubernetes/pki.nix | 31 +++--------- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/nixos/modules/services/cluster/kubernetes/flannel.nix b/nixos/modules/services/cluster/kubernetes/flannel.nix index 4c5fe7559eb..e79fbcb6200 100644 --- a/nixos/modules/services/cluster/kubernetes/flannel.nix +++ b/nixos/modules/services/cluster/kubernetes/flannel.nix @@ -24,16 +24,26 @@ in ###### interface options.services.kubernetes.flannel = { enable = mkEnableOption "enable flannel networking"; + kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes flannel"; }; ###### implementation - config = mkIf cfg.enable { + config = let + + flannelPaths = filter (a: a != null) [ + cfg.kubeconfig.caFile + cfg.kubeconfig.certFile + cfg.kubeconfig.keyFile + ]; + kubeconfig = top.lib.mkKubeConfig "flannel" cfg.kubeconfig; + + in mkIf cfg.enable { services.flannel = { enable = mkDefault true; network = mkDefault top.clusterCidr; - inherit storageBackend; - nodeName = config.services.kubernetes.kubelet.hostname; + inherit storageBackend kubeconfig; + nodeName = top.kubelet.hostname; }; services.kubernetes.kubelet = { @@ -79,16 +89,35 @@ in wantedBy = [ "flannel.target" ]; after = [ "kubelet.target" ]; before = [ "flannel.target" ]; - path = [ pkgs.iptables ]; - preStart = '' - ${top.lib.mkWaitCurl ( with config.systemd.services.flannel; { - path = "/api/v1/nodes"; - cacert = top.caFile; - args = "-o - | grep podCIDR >/dev/null"; - } // optionalAttrs (environment ? cert) { inherit (environment) cert key; })} + path = with pkgs; [ iptables kubectl ]; + environment.KUBECONFIG = kubeconfig; + preStart = let + args = [ + "--selector=kubernetes.io/hostname=${top.kubelet.hostname}" + # flannel exits if node is not registered yet, before that there is no podCIDR + "--output=jsonpath={.items[0].spec.podCIDR}" + # if jsonpath cannot be resolved exit with status 1 + "--allow-missing-template-keys=false" + ]; + in '' + until kubectl get nodes ${concatStringsSep " " args} 2>/dev/null; do + echo Waiting for ${top.kubelet.hostname} to be RegisteredNode + sleep 1 + done ''; + unitConfig.ConditionPathExists = flannelPaths; }; + systemd.paths.flannel = { + wantedBy = [ "flannel.service" ]; + pathConfig = { + PathExists = flannelPaths; + PathChanged = flannelPaths; + }; + }; + + services.kubernetes.flannel.kubeconfig.server = mkDefault top.apiserverAddress; + systemd.services.docker = { environment.DOCKER_OPTS = "-b none"; serviceConfig.EnvironmentFile = "-/run/flannel/docker"; diff --git a/nixos/modules/services/cluster/kubernetes/pki.nix b/nixos/modules/services/cluster/kubernetes/pki.nix index 3c7af73e0b9..2e79e7590e3 100644 --- a/nixos/modules/services/cluster/kubernetes/pki.nix +++ b/nixos/modules/services/cluster/kubernetes/pki.nix @@ -124,10 +124,6 @@ in top.caFile certmgrAPITokenPath ]; - flannelPaths = [ - cfg.certs.flannelClient.cert - cfg.certs.flannelClient.key - ]; proxyPaths = mkIf top.proxy.enable [ cfg.certs.kubeProxyClient.cert cfg.certs.kubeProxyClient.key @@ -375,27 +371,6 @@ in 127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local ''; - services.flannel = with cfg.certs.flannelClient; { - kubeconfig = top.lib.mkKubeConfig "flannel" { - server = top.apiserverAddress; - certFile = cert; - keyFile = key; - }; - }; - - systemd.services.flannel = mkIf top.flannel.enable { - environment = { inherit (top.pki.certs.flannelClient) cert key; }; - unitConfig.ConditionPathExists = flannelPaths; - }; - - systemd.paths.flannel = mkIf top.flannel.enable { - wantedBy = [ "flannel.service" ]; - pathConfig = { - PathExists = flannelPaths; - PathChanged = flannelPaths; - }; - }; - systemd.services.kube-proxy = mkIf top.proxy.enable { environment = { inherit (top.pki.certs.kubeProxyClient) cert key; }; unitConfig.ConditionPathExists = proxyPaths; @@ -453,6 +428,12 @@ in keyFile = mkDefault key; }; }; + flannel = mkIf top.flannel.enable { + kubeconfig = with cfg.certs.flannelClient; { + certFile = cert; + keyFile = key; + }; + }; scheduler = mkIf top.scheduler.enable { kubeconfig = with cfg.certs.schedulerClient; { certFile = mkDefault cert;