k3s: init at 1.17.3+k3s1

This packages k3s as a single self-contained binary (as upstream k3s
does), but without having to download any impure already-built binaries.

The upstream packaging scripts are used. Due to k3s's rather complicated
packaging arrangement, this ends up being a rather long derivation.
This commit is contained in:
Euan Kemp 2020-03-22 22:26:38 -07:00 committed by Frederik Rietdijk
parent d6a8d0ca5b
commit e370711ad0
4 changed files with 315 additions and 0 deletions

View file

@ -2404,6 +2404,12 @@
fingerprint = "67FE 98F2 8C44 CF22 1828 E12F D57E FA62 5C9A 925F";
}];
};
euank = {
email = "euank-nixpkg@euank.com";
github = "euank";
githubId = 2147649;
name = "Euan Kemp";
};
evanjs = {
email = "evanjsx@gmail.com";
github = "evanjs";

View file

@ -0,0 +1,233 @@
with import <nixpkgs> {};
{ stdenv, lib, makeWrapper, fetchFromGitHub, fetchurl, fetchzip }:
with lib;
# k3s is a kinda weird derivation. One of the main points of k3s is the
# simplicity of it being one binary that can perform several tasks.
# However, when you have a good package manager (like nix), that doesn't
# actually make much of a difference; you don't really care if it's one binary
# or 10 since with a good package manager, installing and running it is
# identical.
# Since upstream k3s packages itself as one large binary with several
# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s
# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with
# some exceptions.
# K3s also carries patches to some packages (such as containerd and cni
# plugins), so we intentionally use the k3s versions of those binaries for k3s,
# even if the upstream version of those binaries exist in nixpkgs already. In
# the end, that means we have a thick k3s binary that behaves like the upstream
# one for the most part.
# However, k3s also bundles several pieces of unpatched software, from the
# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc.
# Those pieces of software we entirely ignore upstream's handling of, and just
# make sure they're in the path if desired.
let
k3sVersion = "1.17.3+k3s1"; # k3s git tag
traefikChartVersion = "1.81.0"; # taken from ./scripts/version.sh at the above k3s tag
k3sRootVersion = "0.3.0"; # taken from .s/cripts/version.sh at the above k3s tag
# bundled into the k3s binary
traefikChart = fetchurl {
url = "https://kubernetes-charts.storage.googleapis.com/traefik-${traefikChartVersion}.tgz";
sha256 = "1aqpzgjlvqhil0g3angz94zd4xbl4iq0qmpjcy5aq1xv9qciwdi9";
};
# so, k3s is a complicated thing to package
# This derivation attempts to avoid including any random binaries from the
# internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which
# we don't care about doing, we can add those as build or runtime
# dependencies using a real package manager).
# In addition to those binaries, it's also configuration though (right now
# mostly strongswan configuration), and k3s does use those files.
# As such, we download it in order to grab 'etc' and bundle it into the final
# k3s binary.
k3sRoot = fetchzip {
# Note: marked as apache 2.0 license
url = "https://github.com/rancher/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar";
sha256 = "12xafn5jivl8lqdcs25b28xrc4mf7yf1xif5np169nvvxgvmpdxp";
stripRoot=false;
};
k3sPlugins = buildGoPackage rec {
name = "k3s-cni-plugins";
version = "0.7.6-k3s1"; # from ./scripts/version.sh 'VERSION_CNIPLUGINS'; update when k3s's repo is updated.
goPackagePath = "github.com/containernetworking/plugins";
subPackages = [ "." ];
src = fetchFromGitHub {
owner = "rancher";
repo = "plugins";
rev = "v${version}";
sha256 = "0ax72z1ziann352bp6khfds8vlf3bbkqckrkpx4l4jxgqks45izs";
};
meta = {
description = "k3s-cni-plugins";
license = licenses.asl20;
homepage = https://k3s.io;
maintainers = [];
platforms = platforms.linux;
};
};
# Grab this separately from a build because it's used by both stages of the
# k3s build.
k3sRepo = fetchgit {
url = "https://github.com/rancher/k3s";
rev = "v${k3sVersion}";
leaveDotGit = true; # for version / build date below
sha256 = "0qahyc0mf9glxj49va6d20mcncqg4svfic2iz8b1lqid5c4g68mm";
};
# Stage 1 of the k3s build:
# Let's talk about how k3s is structured.
# One of the ideas of k3s is that there's the single "k3s" binary which can
# do everything you need, from running a k3s server, to being a worker node,
# to running kubectl.
# The way that actually works is that k3s is a single go binary that contains
# a bunch of bindata that it unpacks at runtime into directories (either the
# user's home directory or /var/lib/rancher if run as root).
# This bindata includes both binaries and configuration.
# In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations.
# First, we build all the binaries that get packed into the thick k3s binary
# (and output them from one derivation so they'll all be suitably patched up).
# Then, we bundle those binaries into our thick k3s binary and use that as
# the final single output.
# This approach was chosen because it ensures the bundled binaries all are
# correctly built to run with nix (we can lean on the existing buildGoPackage
# stuff), and we can again lean on that tooling for the final k3s binary too.
# Other alternatives would be to manually run the
# strip/patchelf/remove-references step ourselves in the installPhase of the
# derivation when we've built all the binaries, but haven't bundled them in
# with generated bindata yet.
k3sBuildStage1 = buildGoPackage rec {
name = "k3s-build-1";
version = "${k3sVersion}";
goPackagePath = "github.com/rancher/k3s";
src = k3sRepo;
patches = [ ./patches/00-k3s.patch ];
nativeBuildInputs = [ pkgconfig autoPatchelfHook breakpointHook ];
buildInputs = [ git runc libseccomp ];
buildPhase = ''
pushd go/src/${goPackagePath}
patchShebangs ./scripts/build ./scripts/version.sh
mkdir -p bin
./scripts/build
popd
'';
installPhase = ''
pushd go/src/${goPackagePath}
mkdir -p "$bin/bin"
install -m 0755 -t "$bin/bin" ./bin/*
popd
'';
meta = {
description = "The various binaries that get packaged into the final k3s binary.";
license = licenses.asl20;
homepage = https://k3s.io;
maintainers = [];
platforms = platforms.linux;
};
};
k3sBuild = buildGoPackage rec {
name = "k3s-build";
version = "${k3sVersion}";
goPackagePath = "github.com/rancher/k3s";
src = k3sRepo;
patches = [ ./patches/00-k3s.patch ];
nativeBuildInputs = [ pkgconfig autoPatchelfHook breakpointHook ];
buildInputs = [ git k3sBuildStage1 ];
# In order to build the thick k3s binary (which is what
# ./scripts/package-cli does), we need to get all the binaries that script
# expects in place.
buildPhase = ''
pushd go/src/${goPackagePath}
patchShebangs ./scripts/build ./scripts/version.sh ./scripts/package-cli
mkdir -p bin
install -m 0755 -t ./bin ${k3sBuildStage1}/bin/*
install -m 0755 -T "${k3sPlugins}/bin/plugins" ./bin/cni
# Note: use the already-nixpkgs-bundled k3s rather than the one bundled
# in k3s because the k3s one is completely unmodified from upstream
# (unlike containerd, cni, etc)
install -m 0755 -T "${runc}/bin/runc" ./bin/runc
cp -R "${k3sRoot}/etc" ./etc
mkdir -p "build/static/charts"
cp "${traefikChart}" "build/static/charts/traefik-${traefikChartVersion}.tgz"
./scripts/package-cli
popd
'';
installPhase = ''
pushd go/src/${goPackagePath}
mkdir -p "$bin/bin"
install -m 0755 -t "$bin/bin" ./dist/artifacts/k3s
popd
'';
meta = {
description = "The k3s go binary which is used by the final wrapped output below.";
license = licenses.asl20;
homepage = https://k3s.io;
maintainers = [];
platforms = platforms.linux;
};
};
in
stdenv.mkDerivation rec {
name = "k3s";
# Important utilities used by the kubelet, see
# https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494
# Note the list in that issue is stale and some aren't relevant for k3s.
k3sRuntimeDeps = [
socat iptables iproute bridge-utils ethtool utillinux ipset conntrack-tools
];
buildInputs = [
k3sBuild makeWrapper
] ++ k3sRuntimeDeps;
unpackPhase = "true";
# And, one final derivation (you thought the last one was it, right?)
# We got the binary we wanted above, but it doesn't have all the runtime
# dependencies k8s wants, including mount utilities for kubelet, networking
# tools for cni/kubelet stuff, etc
# Use a wrapper script to reference all the binaries that k3s tries to
# execute, but that we didn't bundle with it.
installPhase = ''
mkdir -p "$out/bin"
makeWrapper ${k3sBuild}/bin/k3s "$out/bin/k3s" \
--prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \
--prefix PATH : "$out/bin"
'';
meta = {
description = "A lightweight Kubernetes distribution.";
license = licenses.asl20;
homepage = https://k3s.io;
maintainers = with maintainers; [ euank ];
platforms = platforms.linux;
};
}

View file

@ -0,0 +1,74 @@
diff --git a/main.go b/main.go
index 62908bb7bb..0527222887 100644
--- a/main.go
+++ b/main.go
@@ -1,5 +1,5 @@
//go:generate go run pkg/codegen/cleanup/main.go
-//go:generate /bin/rm -rf pkg/generated
+//go:generate rm -rf pkg/generated
//go:generate go run pkg/codegen/main.go
//go:generate go fmt pkg/deploy/zz_generated_bindata.go
//go:generate go fmt pkg/static/zz_generated_bindata.go
diff --git a/scripts/build b/scripts/build
index 72d3c07ece..3e5455b262 100755
--- a/scripts/build
+++ b/scripts/build
@@ -10,7 +10,8 @@ PKG_CONTAINERD="github.com/containerd/containerd"
PKG_RANCHER_CONTAINERD="github.com/rancher/containerd"
PKG_CRICTL="github.com/kubernetes-sigs/cri-tools"
-buildDate=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+# Deterministic build date
+buildDate="$(date -d "$(git log -1 --format=%ai)" -u "+%Y-%m-%dT%H:%M:%SZ")"
VENDOR_PREFIX="${PKG}/vendor/"
VERSIONFLAGS="
@@ -82,17 +83,7 @@ cleanup() {
}
INSTALLBIN=$(pwd)/bin
-if [ ! -x ${INSTALLBIN}/cni ]; then
-(
- echo Building cni
- TMPDIR=$(mktemp -d)
- trap cleanup EXIT
- WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins
- git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR
- cd $WORKDIR
- GOPATH=$TMPDIR CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o $INSTALLBIN/cni
-)
-fi
+# skip building cni, use our separately built one
# echo Building agent
# CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go
echo Building server
@@ -108,9 +99,8 @@ ln -s containerd ./bin/ctr
#CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/ctr ./cmd/ctr/main.go
# echo Building containerd
# CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/
-echo Building runc
-make EXTRA_LDFLAGS="-w -s" BUILDTAGS="apparmor seccomp" -C ./vendor/github.com/opencontainers/runc static
-cp -f ./vendor/github.com/opencontainers/runc/runc ./bin/runc
+
+# skip building runc; use our packaged one
echo Building containerd-shim
make -C ./vendor/github.com/containerd/containerd bin/containerd-shim
diff --git a/scripts/package-cli b/scripts/package-cli
index 4c66ce32df..6d1e0c03cb 100755
--- a/scripts/package-cli
+++ b/scripts/package-cli
@@ -55,10 +55,10 @@ LDFLAGS="
-X github.com/rancher/k3s/pkg/version.GitCommit=${COMMIT:0:8}
-w -s
"
-STATIC="-extldflags '-static'"
if [ "$DQLITE" = "true" ]; then
DQLITE_TAGS="dqlite"
fi
-CGO_ENABLED=0 go build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
+go build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS" -o ${CMD_NAME} ./cmd/k3s/main.go
-./scripts/build-upload ${CMD_NAME} ${COMMIT}
+# for nixos, don't upload it
+# ./scripts/build-upload ${CMD_NAME} ${COMMIT}

View file

@ -20159,6 +20159,8 @@ in
boost = boost155.override { enablePython = true; };
};
k3s = callPackage ../applications/networking/cluster/k3s {};
k9copy = libsForQt5.callPackage ../applications/video/k9copy {};
kail = callPackage ../tools/networking/kail { };