Merge pull request #72603 from flokli/ceph-tmpfiles

nixos/ceph: run unprivileged, use state directories, handle non-initialized clusters without config switch
This commit is contained in:
Florian Klink 2019-11-11 13:42:54 +01:00 committed by GitHub
commit 60390c81dc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 74 additions and 94 deletions

View file

@ -9,12 +9,14 @@ let
expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
makeServices = (daemonType: daemonIds: extraServiceConfig:
makeServices = (daemonType: daemonIds:
mkMerge (map (daemonId:
{ "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph extraServiceConfig; })
{ "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; })
daemonIds));
makeService = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {
makeService = (daemonType: daemonId: clusterName: ceph:
let
stateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; in {
enable = true;
description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
@ -22,6 +24,11 @@ let
partOf = [ "ceph-${daemonType}.target" ];
wantedBy = [ "ceph-${daemonType}.target" ];
path = [ pkgs.getopt ];
# Don't start services that are not yet initialized
unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring";
serviceConfig = {
LimitNOFILE = 1048576;
LimitNPROC = 1048576;
@ -34,22 +41,22 @@ let
Restart = "on-failure";
StartLimitBurst = "5";
StartLimitInterval = "30min";
StateDirectory = stateDirectory;
User = "ceph";
Group = if daemonType == "osd" then "disk" else "ceph";
ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
-f --cluster ${clusterName} --id ${daemonId} --setuser ceph \
--setgroup ${if daemonType == "osd" then "disk" else "ceph"}'';
} // extraServiceConfig
// optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh \
--id ${daemonId} --cluster ${clusterName}''; };
} // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) {
preStart = ''
daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"
if [ ! -d $daemonPath ]; then
mkdir -m 755 -p $daemonPath
chown -R ceph:ceph $daemonPath
fi
'';
} // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; }
);
-f --cluster ${clusterName} --id ${daemonId}'';
} // optionalAttrs (daemonType == "osd") {
ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}'';
StartLimitBurst = "30";
RestartSec = "20s";
PrivateDevices = "no"; # osd needs disk access
} // optionalAttrs ( daemonType == "mon") {
RestartSec = "10";
} // optionalAttrs (lib.elem daemonType ["mgr" "mds"]) {
StartLimitBurst = "3";
};
});
makeTarget = (daemonType:
{
@ -58,6 +65,7 @@ let
partOf = [ "ceph.target" ];
wantedBy = [ "ceph.target" ];
before = [ "ceph.target" ];
unitConfig.StopWhenUnneeded = true;
};
}
);
@ -377,22 +385,22 @@ in
systemd.services = let
services = []
++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons { RestartSec = "10"; })
++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons { StartLimitBurst = "3"; })
++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons { StartLimitBurst = "30";
RestartSec = "20s";
PrivateDevices = "no"; # osd needs disk access
})
++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons { })
++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });
++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons)
++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons)
++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons)
++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons)
++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons);
in
mkMerge services;
systemd.targets = let
targets = [
{ ceph = { description = "Ceph target allowing to start/stop all ceph service instances at once";
wantedBy = [ "multi-user.target" ]; }; }
] ++ optional cfg.mon.enable (makeTarget "mon")
{ ceph = {
description = "Ceph target allowing to start/stop all ceph service instances at once";
wantedBy = [ "multi-user.target" ];
unitConfig.StopWhenUnneeded = true;
}; } ]
++ optional cfg.mon.enable (makeTarget "mon")
++ optional cfg.mds.enable (makeTarget "mds")
++ optional cfg.osd.enable (makeTarget "osd")
++ optional cfg.rgw.enable (makeTarget "rgw")
@ -401,7 +409,11 @@ in
mkMerge targets;
systemd.tmpfiles.rules = [
"d /etc/ceph - ceph ceph - -"
"d /run/ceph 0770 ceph ceph -"
];
"d /var/lib/ceph - ceph ceph - -"]
++ optionals cfg.mgr.enable [ "d /var/lib/ceph/mgr - ceph ceph - -"]
++ optionals cfg.mon.enable [ "d /var/lib/ceph/mon - ceph ceph - -"]
++ optionals cfg.osd.enable [ "d /var/lib/ceph/osd - ceph ceph - -"];
};
}

View file

@ -49,9 +49,6 @@ let
boot.kernelModules = [ "xfs" ];
services.ceph = cephConfig;
# So that we don't have to battle systemd when bootstraping
systemd.targets.ceph.wantedBy = lib.mkForce [];
};
networkMonA = {
@ -107,6 +104,10 @@ let
};
}; };
# Following deployment is based on the manual deployment described here:
# https://docs.ceph.com/docs/master/install/manual-deployment/
# For other ways to deploy a ceph cluster, look at the documentation at
# https://docs.ceph.com/docs/master/
testscript = { ... }: ''
startAll;
@ -114,27 +115,6 @@ let
$osd0->waitForUnit("network.target");
$osd1->waitForUnit("network.target");
# Create the ceph-related directories
$monA->mustSucceed(
"mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}",
"mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}",
"chown ceph:ceph -R /var/lib/ceph/",
"mkdir -p /etc/ceph",
"chown ceph:ceph -R /etc/ceph"
);
$osd0->mustSucceed(
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"chown ceph:ceph -R /var/lib/ceph/",
"mkdir -p /etc/ceph",
"chown ceph:ceph -R /etc/ceph"
);
$osd1->mustSucceed(
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"chown ceph:ceph -R /var/lib/ceph/",
"mkdir -p /etc/ceph",
"chown ceph:ceph -R /etc/ceph"
);
# Bootstrap ceph-mon daemon
$monA->mustSucceed(
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
@ -142,6 +122,7 @@ let
"sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
"monmaptool --create --add ${cfg.monA.name} ${cfg.monA.ip} --fsid ${cfg.clusterId} /tmp/monmap",
"sudo -u ceph ceph-mon --mkfs -i ${cfg.monA.name} --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
"sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
"sudo -u ceph touch /var/lib/ceph/mon/ceph-${cfg.monA.name}/done",
"systemctl start ceph-mon-${cfg.monA.name}"
);
@ -168,12 +149,14 @@ let
# Bootstrap both OSDs
$osd0->mustSucceed(
"mkfs.xfs /dev/vdb",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
"echo '{\"cephx_secret\": \"${cfg.osd0.key}\"}' | ceph osd new ${cfg.osd0.uuid} -i -",
);
$osd1->mustSucceed(
"mkfs.xfs /dev/vdb",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
"echo '{\"cephx_secret\": \"${cfg.osd1.key}\"}' | ceph osd new ${cfg.osd1.uuid} -i -"
@ -209,22 +192,17 @@ let
"ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it"
);
# As we disable the target in the config, we still want to test that it works as intended
$osd0->mustSucceed("systemctl stop ceph-osd-${cfg.osd0.name}");
$osd1->mustSucceed("systemctl stop ceph-osd-${cfg.osd1.name}");
$monA->mustSucceed(
"systemctl stop ceph-mgr-${cfg.monA.name}",
"systemctl stop ceph-mon-${cfg.monA.name}"
);
$monA->succeed("systemctl start ceph.target");
$monA->waitForUnit("ceph-mon-${cfg.monA.name}");
$monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
$osd0->succeed("systemctl start ceph.target");
$osd0->waitForUnit("ceph-osd-${cfg.osd0.name}");
$osd1->succeed("systemctl start ceph.target");
$osd1->waitForUnit("ceph-osd-${cfg.osd1.name}");
# Shut down ceph on all machines in a very unpolite way
$monA->crash;
$osd0->crash;
$osd1->crash;
# Start it up
$osd0->start;
$osd1->start;
$monA->start;
# Ensure the cluster comes back up again
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
$monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
$monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");

View file

@ -46,9 +46,6 @@ let
boot.kernelModules = [ "xfs" ];
services.ceph = cephConfig;
# So that we don't have to battle systemd when bootstraping
systemd.targets.ceph.wantedBy = lib.mkForce [];
};
networkMonA = {
@ -72,22 +69,15 @@ let
};
}; };
# Following deployment is based on the manual deployment described here:
# https://docs.ceph.com/docs/master/install/manual-deployment/
# For other ways to deploy a ceph cluster, look at the documentation at
# https://docs.ceph.com/docs/master/
testscript = { ... }: ''
startAll;
$monA->waitForUnit("network.target");
# Create the ceph-related directories
$monA->mustSucceed(
"mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}",
"mkdir -p /var/lib/ceph/mon/ceph-${cfg.monA.name}",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"mkdir -p /etc/ceph",
"chown ceph:ceph -R /etc/ceph",
"chown ceph:ceph -R /var/lib/ceph/",
);
# Bootstrap ceph-mon daemon
$monA->mustSucceed(
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
@ -104,8 +94,9 @@ let
# Can't check ceph status until a mon is up
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
# Start the ceph-mgr daemon, it has no deps and hardly any setup
# Start the ceph-mgr daemon, after copying in the keyring
$monA->mustSucceed(
"sudo -u ceph mkdir -p /var/lib/ceph/mgr/ceph-${cfg.monA.name}/",
"ceph auth get-or-create mgr.${cfg.monA.name} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-${cfg.monA.name}/keyring",
"systemctl start ceph-mgr-${cfg.monA.name}"
);
@ -117,7 +108,9 @@ let
$monA->mustSucceed(
"mkfs.xfs /dev/vdb",
"mkfs.xfs /dev/vdc",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"mount /dev/vdb /var/lib/ceph/osd/ceph-${cfg.osd0.name}",
"mkdir -p /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"mount /dev/vdc /var/lib/ceph/osd/ceph-${cfg.osd1.name}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd0.name}/keyring --name osd.${cfg.osd0.name} --add-key ${cfg.osd0.key}",
"ceph-authtool --create-keyring /var/lib/ceph/osd/ceph-${cfg.osd1.name}/keyring --name osd.${cfg.osd1.name} --add-key ${cfg.osd1.key}",
@ -159,20 +152,17 @@ let
"ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it"
);
# As we disable the target in the config, we still want to test that it works as intended
$monA->mustSucceed(
"systemctl stop ceph-osd-${cfg.osd0.name}",
"systemctl stop ceph-osd-${cfg.osd1.name}",
"systemctl stop ceph-mgr-${cfg.monA.name}",
"systemctl stop ceph-mon-${cfg.monA.name}"
);
# Shut down ceph by stopping ceph.target.
$monA->mustSucceed("systemctl stop ceph.target");
# Start it up
$monA->succeed("systemctl start ceph.target");
$monA->waitForUnit("ceph-mon-${cfg.monA.name}");
$monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
$monA->waitForUnit("ceph-osd-${cfg.osd0.name}");
$monA->waitForUnit("ceph-osd-${cfg.osd1.name}");
# Ensure the cluster comes back up again
$monA->succeed("ceph -s | grep 'mon: 1 daemons'");
$monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
$monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");