Merge commit a1dd396cc02922372314c35c8035a38bfeea08df of branch 'nix'.
This commit is contained in:
parent
828c0bec6b
commit
15ddeff532
@ -13,6 +13,7 @@
|
|||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
|
||||||
|
#include <limits.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
#include <sys/time.h>
|
#include <sys/time.h>
|
||||||
#include <sys/wait.h>
|
#include <sys/wait.h>
|
||||||
|
@ -48,7 +48,7 @@ static Path parsePath(std::istream & str)
|
|||||||
{
|
{
|
||||||
string s = parseString(str);
|
string s = parseString(str);
|
||||||
if (s.size() == 0 || s[0] != '/')
|
if (s.size() == 0 || s[0] != '/')
|
||||||
throw Error(format("bad path `%1%' in derivation") % s);
|
throw FormatError(format("bad path `%1%' in derivation") % s);
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -62,7 +62,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
Derivation parseDerivation(const string & s)
|
static Derivation parseDerivation(const string & s)
|
||||||
{
|
{
|
||||||
Derivation drv;
|
Derivation drv;
|
||||||
std::istringstream str(s);
|
std::istringstream str(s);
|
||||||
@ -112,6 +112,16 @@ Derivation parseDerivation(const string & s)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Derivation readDerivation(const Path & drvPath)
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
return parseDerivation(readFile(drvPath));
|
||||||
|
} catch (FormatError & e) {
|
||||||
|
throw Error(format("error parsing derivation `%1%': %2%") % drvPath % e.msg());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
static void printString(string & res, const string & s)
|
static void printString(string & res, const string & s)
|
||||||
{
|
{
|
||||||
res += '"';
|
res += '"';
|
||||||
@ -240,7 +250,7 @@ Hash hashDerivationModulo(StoreAPI & store, Derivation drv)
|
|||||||
Hash h = drvHashes[i->first];
|
Hash h = drvHashes[i->first];
|
||||||
if (h.type == htUnknown) {
|
if (h.type == htUnknown) {
|
||||||
assert(store.isValidPath(i->first));
|
assert(store.isValidPath(i->first));
|
||||||
Derivation drv2 = parseDerivation(readFile(i->first));
|
Derivation drv2 = readDerivation(i->first);
|
||||||
h = hashDerivationModulo(store, drv2);
|
h = hashDerivationModulo(store, drv2);
|
||||||
drvHashes[i->first] = h;
|
drvHashes[i->first] = h;
|
||||||
}
|
}
|
||||||
|
@ -59,8 +59,8 @@ class StoreAPI;
|
|||||||
Path writeDerivation(StoreAPI & store,
|
Path writeDerivation(StoreAPI & store,
|
||||||
const Derivation & drv, const string & name, bool repair = false);
|
const Derivation & drv, const string & name, bool repair = false);
|
||||||
|
|
||||||
/* Parse a derivation. */
|
/* Read a derivation from a file. */
|
||||||
Derivation parseDerivation(const string & s);
|
Derivation readDerivation(const Path & drvPath);
|
||||||
|
|
||||||
/* Print a derivation. */
|
/* Print a derivation. */
|
||||||
string unparseDerivation(const Derivation & drv);
|
string unparseDerivation(const Derivation & drv);
|
||||||
|
@ -20,6 +20,7 @@
|
|||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
#include <grp.h>
|
||||||
|
|
||||||
#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H
|
#if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H
|
||||||
#include <sched.h>
|
#include <sched.h>
|
||||||
@ -237,7 +238,7 @@ LocalStore::LocalStore(bool reserveSpace)
|
|||||||
makeStoreWritable();
|
makeStoreWritable();
|
||||||
createDirs(linksDir = settings.nixStore + "/.links");
|
createDirs(linksDir = settings.nixStore + "/.links");
|
||||||
Path profilesDir = settings.nixStateDir + "/profiles";
|
Path profilesDir = settings.nixStateDir + "/profiles";
|
||||||
createDirs(settings.nixStateDir + "/profiles");
|
createDirs(profilesDir);
|
||||||
createDirs(settings.nixStateDir + "/temproots");
|
createDirs(settings.nixStateDir + "/temproots");
|
||||||
createDirs(settings.nixDBPath);
|
createDirs(settings.nixDBPath);
|
||||||
Path gcRootsDir = settings.nixStateDir + "/gcroots";
|
Path gcRootsDir = settings.nixStateDir + "/gcroots";
|
||||||
@ -246,6 +247,32 @@ LocalStore::LocalStore(bool reserveSpace)
|
|||||||
createSymlink(profilesDir, gcRootsDir + "/profiles");
|
createSymlink(profilesDir, gcRootsDir + "/profiles");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Optionally, create directories and set permissions for a
|
||||||
|
multi-user install. */
|
||||||
|
if (getuid() == 0 && settings.buildUsersGroup != "") {
|
||||||
|
|
||||||
|
Path perUserDir = profilesDir + "/per-user";
|
||||||
|
createDirs(perUserDir);
|
||||||
|
if (chmod(perUserDir.c_str(), 01777) == -1)
|
||||||
|
throw SysError(format("could not set permissions on `%1%' to 1777") % perUserDir);
|
||||||
|
|
||||||
|
struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
|
||||||
|
if (!gr)
|
||||||
|
throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
|
||||||
|
% settings.buildUsersGroup);
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
if (stat(settings.nixStore.c_str(), &st))
|
||||||
|
throw SysError(format("getting attributes of path `%1%'") % settings.nixStore);
|
||||||
|
|
||||||
|
if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != 01775) {
|
||||||
|
if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
|
||||||
|
throw SysError(format("changing ownership of path `%1%'") % settings.nixStore);
|
||||||
|
if (chmod(settings.nixStore.c_str(), 01775) == -1)
|
||||||
|
throw SysError(format("changing permissions on path `%1%'") % settings.nixStore);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
checkStoreNotSymlink();
|
checkStoreNotSymlink();
|
||||||
|
|
||||||
/* We can't open a SQLite database if the disk is full. Since
|
/* We can't open a SQLite database if the disk is full. Since
|
||||||
@ -661,7 +688,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che
|
|||||||
efficiently query whether a path is an output of some
|
efficiently query whether a path is an output of some
|
||||||
derivation. */
|
derivation. */
|
||||||
if (isDerivation(info.path)) {
|
if (isDerivation(info.path)) {
|
||||||
Derivation drv = parseDerivation(readFile(info.path));
|
Derivation drv = readDerivation(info.path);
|
||||||
|
|
||||||
/* Verify that the output paths in the derivation are correct
|
/* Verify that the output paths in the derivation are correct
|
||||||
(i.e., follow the scheme for computing output paths from
|
(i.e., follow the scheme for computing output paths from
|
||||||
@ -1290,7 +1317,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
|||||||
if (isDerivation(i->path)) {
|
if (isDerivation(i->path)) {
|
||||||
// FIXME: inefficient; we already loaded the
|
// FIXME: inefficient; we already loaded the
|
||||||
// derivation in addValidPath().
|
// derivation in addValidPath().
|
||||||
Derivation drv = parseDerivation(readFile(i->path));
|
Derivation drv = readDerivation(i->path);
|
||||||
checkDerivationOutputs(i->path, drv);
|
checkDerivationOutputs(i->path, drv);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -6,6 +6,11 @@
|
|||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
#include "pathlocks.hh"
|
#include "pathlocks.hh"
|
||||||
|
|
||||||
|
#if HAVE_TR1_UNORDERED_SET
|
||||||
|
#include <tr1/unordered_set>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class sqlite3;
|
class sqlite3;
|
||||||
class sqlite3_stmt;
|
class sqlite3_stmt;
|
||||||
@ -29,14 +34,12 @@ struct Derivation;
|
|||||||
|
|
||||||
struct OptimiseStats
|
struct OptimiseStats
|
||||||
{
|
{
|
||||||
unsigned long totalFiles;
|
|
||||||
unsigned long sameContents;
|
|
||||||
unsigned long filesLinked;
|
unsigned long filesLinked;
|
||||||
unsigned long long bytesFreed;
|
unsigned long long bytesFreed;
|
||||||
unsigned long long blocksFreed;
|
unsigned long long blocksFreed;
|
||||||
OptimiseStats()
|
OptimiseStats()
|
||||||
{
|
{
|
||||||
totalFiles = sameContents = filesLinked = 0;
|
filesLinked = 0;
|
||||||
bytesFreed = blocksFreed = 0;
|
bytesFreed = blocksFreed = 0;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -303,7 +306,15 @@ private:
|
|||||||
|
|
||||||
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
|
void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
|
||||||
|
|
||||||
void optimisePath_(OptimiseStats & stats, const Path & path);
|
#if HAVE_TR1_UNORDERED_SET
|
||||||
|
typedef std::tr1::unordered_set<ino_t> InodeHash;
|
||||||
|
#else
|
||||||
|
typedef std::set<ino_t> InodeHash;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
InodeHash loadInodeHash();
|
||||||
|
Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
|
||||||
|
void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash);
|
||||||
|
|
||||||
// Internal versions that are not wrapped in retry_sqlite.
|
// Internal versions that are not wrapped in retry_sqlite.
|
||||||
bool isValidPath_(const Path & path);
|
bool isValidPath_(const Path & path);
|
||||||
|
@ -11,7 +11,7 @@ Derivation derivationFromPath(StoreAPI & store, const Path & drvPath)
|
|||||||
{
|
{
|
||||||
assertStorePath(drvPath);
|
assertStorePath(drvPath);
|
||||||
store.ensurePath(drvPath);
|
store.ensurePath(drvPath);
|
||||||
return parseDerivation(readFile(drvPath));
|
return readDerivation(drvPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -40,18 +40,66 @@ struct MakeReadOnly
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
LocalStore::InodeHash LocalStore::loadInodeHash()
|
||||||
|
{
|
||||||
|
printMsg(lvlDebug, "loading hash inodes in memory");
|
||||||
|
InodeHash inodeHash;
|
||||||
|
|
||||||
|
AutoCloseDir dir = opendir(linksDir.c_str());
|
||||||
|
if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);
|
||||||
|
|
||||||
|
struct dirent * dirent;
|
||||||
|
while (errno = 0, dirent = readdir(dir)) { /* sic */
|
||||||
|
checkInterrupt();
|
||||||
|
// We don't care if we hit non-hash files, anything goes
|
||||||
|
inodeHash.insert(dirent->d_ino);
|
||||||
|
}
|
||||||
|
if (errno) throw SysError(format("reading directory `%1%'") % linksDir);
|
||||||
|
|
||||||
|
printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
|
||||||
|
|
||||||
|
return inodeHash;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash)
|
||||||
|
{
|
||||||
|
Strings names;
|
||||||
|
|
||||||
|
AutoCloseDir dir = opendir(path.c_str());
|
||||||
|
if (!dir) throw SysError(format("opening directory `%1%'") % path);
|
||||||
|
|
||||||
|
struct dirent * dirent;
|
||||||
|
while (errno = 0, dirent = readdir(dir)) { /* sic */
|
||||||
|
checkInterrupt();
|
||||||
|
|
||||||
|
if (inodeHash.count(dirent->d_ino)) {
|
||||||
|
printMsg(lvlDebug, format("`%1%' is already linked") % dirent->d_name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
string name = dirent->d_name;
|
||||||
|
if (name == "." || name == "..") continue;
|
||||||
|
names.push_back(name);
|
||||||
|
}
|
||||||
|
if (errno) throw SysError(format("reading directory `%1%'") % path);
|
||||||
|
|
||||||
|
return names;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash)
|
||||||
{
|
{
|
||||||
checkInterrupt();
|
checkInterrupt();
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (lstat(path.c_str(), &st))
|
if (lstat(path.c_str(), &st))
|
||||||
throw SysError(format("getting attributes of path `%1%'") % path);
|
throw SysError(format("getting attributes of path `%1%'") % path);
|
||||||
|
|
||||||
if (S_ISDIR(st.st_mode)) {
|
if (S_ISDIR(st.st_mode)) {
|
||||||
Strings names = readDirectory(path);
|
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
|
||||||
foreach (Strings::iterator, i, names)
|
foreach (Strings::iterator, i, names)
|
||||||
optimisePath_(stats, path + "/" + *i);
|
optimisePath_(stats, path + "/" + *i, inodeHash);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -71,6 +119,12 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* This can still happen on top-level files */
|
||||||
|
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
|
||||||
|
printMsg(lvlDebug, format("`%1%' is already linked, with %2% other file(s).") % path % (st.st_nlink - 2));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
/* Hash the file. Note that hashPath() returns the hash over the
|
/* Hash the file. Note that hashPath() returns the hash over the
|
||||||
NAR serialisation, which includes the execute bit on the file.
|
NAR serialisation, which includes the execute bit on the file.
|
||||||
Thus, executable and non-executable files with the same
|
Thus, executable and non-executable files with the same
|
||||||
@ -81,7 +135,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
|||||||
contents of the symlink (i.e. the result of readlink()), not
|
contents of the symlink (i.e. the result of readlink()), not
|
||||||
the contents of the target (which may not even exist). */
|
the contents of the target (which may not even exist). */
|
||||||
Hash hash = hashPath(htSHA256, path).first;
|
Hash hash = hashPath(htSHA256, path).first;
|
||||||
stats.totalFiles++;
|
|
||||||
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
|
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
|
||||||
|
|
||||||
/* Check if this is a known hash. */
|
/* Check if this is a known hash. */
|
||||||
@ -89,7 +142,10 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
|||||||
|
|
||||||
if (!pathExists(linkPath)) {
|
if (!pathExists(linkPath)) {
|
||||||
/* Nope, create a hard link in the links directory. */
|
/* Nope, create a hard link in the links directory. */
|
||||||
if (link(path.c_str(), linkPath.c_str()) == 0) return;
|
if (link(path.c_str(), linkPath.c_str()) == 0) {
|
||||||
|
inodeHash.insert(st.st_ino);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (errno != EEXIST)
|
if (errno != EEXIST)
|
||||||
throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
|
throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
|
||||||
/* Fall through if another process created ‘linkPath’ before
|
/* Fall through if another process created ‘linkPath’ before
|
||||||
@ -102,7 +158,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
|||||||
if (lstat(linkPath.c_str(), &stLink))
|
if (lstat(linkPath.c_str(), &stLink))
|
||||||
throw SysError(format("getting attributes of path `%1%'") % linkPath);
|
throw SysError(format("getting attributes of path `%1%'") % linkPath);
|
||||||
|
|
||||||
stats.sameContents++;
|
|
||||||
if (st.st_ino == stLink.st_ino) {
|
if (st.st_ino == stLink.st_ino) {
|
||||||
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
|
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
|
||||||
return;
|
return;
|
||||||
@ -160,12 +215,13 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
|
|||||||
void LocalStore::optimiseStore(OptimiseStats & stats)
|
void LocalStore::optimiseStore(OptimiseStats & stats)
|
||||||
{
|
{
|
||||||
PathSet paths = queryAllValidPaths();
|
PathSet paths = queryAllValidPaths();
|
||||||
|
InodeHash inodeHash = loadInodeHash();
|
||||||
|
|
||||||
foreach (PathSet::iterator, i, paths) {
|
foreach (PathSet::iterator, i, paths) {
|
||||||
addTempRoot(*i);
|
addTempRoot(*i);
|
||||||
if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
|
if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
|
||||||
startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
|
startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
|
||||||
optimisePath_(stats, *i);
|
optimisePath_(stats, *i, inodeHash);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -173,7 +229,9 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
|
|||||||
void LocalStore::optimisePath(const Path & path)
|
void LocalStore::optimisePath(const Path & path)
|
||||||
{
|
{
|
||||||
OptimiseStats stats;
|
OptimiseStats stats;
|
||||||
if (settings.autoOptimiseStore) optimisePath_(stats, path);
|
InodeHash inodeHash;
|
||||||
|
|
||||||
|
if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -104,7 +104,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
|
|||||||
writeString(readLink(path), sink);
|
writeString(readLink(path), sink);
|
||||||
}
|
}
|
||||||
|
|
||||||
else throw Error(format("file `%1%' has an unknown type") % path);
|
else throw Error(format("file `%1%' has an unsupported type") % path);
|
||||||
|
|
||||||
writeString(")", sink);
|
writeString(")", sink);
|
||||||
}
|
}
|
||||||
|
@ -1041,7 +1041,7 @@ void expect(std::istream & str, const string & s)
|
|||||||
char s2[s.size()];
|
char s2[s.size()];
|
||||||
str.read(s2, s.size());
|
str.read(s2, s.size());
|
||||||
if (string(s2, s.size()) != s)
|
if (string(s2, s.size()) != s)
|
||||||
throw Error(format("expected string `%1%'") % s);
|
throw FormatError(format("expected string `%1%'") % s);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -326,6 +326,8 @@ bool hasSuffix(const string & s, const string & suffix);
|
|||||||
/* Read string `s' from stream `str'. */
|
/* Read string `s' from stream `str'. */
|
||||||
void expect(std::istream & str, const string & s);
|
void expect(std::istream & str, const string & s);
|
||||||
|
|
||||||
|
MakeError(FormatError, Error)
|
||||||
|
|
||||||
|
|
||||||
/* Read a C-style string from stream `str'. */
|
/* Read a C-style string from stream `str'. */
|
||||||
string parseString(std::istream & str);
|
string parseString(std::istream & str);
|
||||||
|
Loading…
Reference in New Issue
Block a user