Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
openSUSE:Step:15-SP4
rpm
ndb-backport.diff
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File ndb-backport.diff of Package rpm
--- ./lib/backend/ndb/glue.c.orig 2020-04-28 12:50:11.812398993 +0000 +++ ./lib/backend/ndb/glue.c 2021-04-21 13:25:45.427465231 +0000 @@ -30,6 +30,7 @@ struct ndbEnv_s { rpmpkgdb pkgdb; rpmxdb xdb; int refs; + int dofsync; unsigned int hdrNum; void *data; @@ -51,15 +52,17 @@ static void closeEnv(rpmdb rdb) if (ndbenv->data) free(ndbenv->data); free(ndbenv); + rdb->db_dbenv = 0; } - rdb->db_dbenv = 0; } static struct ndbEnv_s *openEnv(rpmdb rdb) { struct ndbEnv_s *ndbenv = rdb->db_dbenv; - if (!ndbenv) + if (!ndbenv) { rdb->db_dbenv = ndbenv = xcalloc(1, sizeof(struct ndbEnv_s)); + ndbenv->dofsync = 1; + } ndbenv->refs++; return ndbenv; } @@ -77,6 +80,31 @@ static int ndb_Close(dbiIndex dbi, unsig return 0; } +static void ndb_CheckIndexSync(rpmpkgdb pkgdb, rpmxdb xdb) +{ + unsigned int generation, xdb_generation; + if (!pkgdb || !xdb) + return; + if (rpmpkgLock(pkgdb, 0)) + return; + if (rpmpkgGeneration(pkgdb, &generation)) { + rpmpkgUnlock(pkgdb, 0); + return; + } + if (!rpmxdbGetUserGeneration(xdb, &xdb_generation) && generation == xdb_generation) { + rpmpkgUnlock(pkgdb, 0); + return; + } + rpmpkgUnlock(pkgdb, 0); + /* index corrupt or with different generation */ + if (rpmxdbIsRdonly(xdb)) { + rpmlog(RPMLOG_WARNING, _("Detected outdated index databases\n")); + } else { + rpmlog(RPMLOG_WARNING, _("Rebuilding outdated index databases\n")); + rpmxdbDelAllBlobs(xdb); + } +} + static int ndb_Open(rpmdb rdb, rpmDbiTagVal rpmtag, dbiIndex * dbip, int flags) { const char *dbhome = rpmdbHome(rdb); @@ -114,6 +142,7 @@ static int ndb_Open(rpmdb rdb, rpmDbiTag } free(path); dbi->dbi_db = ndbenv->pkgdb = pkgdb; + rpmpkgSetFsync(pkgdb, ndbenv->dofsync); if ((oflags & (O_RDWR | O_RDONLY)) == O_RDONLY) dbi->dbi_flags |= DBI_RDONLY; @@ -126,6 +155,7 @@ static int ndb_Open(rpmdb rdb, rpmDbiTag } if (!ndbenv->xdb) { char *path = rstrscat(NULL, dbhome, "/Index.db", NULL); + int created = 0; rpmlog(RPMLOG_DEBUG, "opening db index %s mode=0x%x\n", path, rdb->db_mode); /* Open indexes readwrite if possible */ @@ -140,6 +170,7 @@ static int ndb_Open(rpmdb rdb, rpmDbiTag } else if (rc && errno == ENOENT) { ioflags = O_CREAT|O_RDWR; rc = rpmxdbOpen(&ndbenv->xdb, rdb->db_pkgs->dbi_db, path, ioflags, 0666); + created = 1; } if (rc) { perror("rpmxdbOpen"); @@ -148,6 +179,9 @@ static int ndb_Open(rpmdb rdb, rpmDbiTag return 1; } free(path); + rpmxdbSetFsync(ndbenv->xdb, ndbenv->dofsync); + if (!created) + ndb_CheckIndexSync(ndbenv->pkgdb, ndbenv->xdb); } if (rpmxdbLookupBlob(ndbenv->xdb, &id, rpmtag, 0, 0) == RPMRC_NOTFOUND) { dbi->dbi_flags |= DBI_CREATED; @@ -174,11 +208,25 @@ static int ndb_Open(rpmdb rdb, rpmDbiTag static int ndb_Verify(dbiIndex dbi, unsigned int flags) { - return 1; + int rc; + if (dbi->dbi_type == DBI_PRIMARY) { + rc = rpmpkgVerify(dbi->dbi_db); + } else { + rc = 0; /* cannot verify the index databases */ + } + return rc; } static void ndb_SetFSync(rpmdb rdb, int enable) { + struct ndbEnv_s *ndbenv = rdb->db_dbenv; + if (ndbenv) { + ndbenv->dofsync = enable; + if (ndbenv->pkgdb) + rpmpkgSetFsync(ndbenv->pkgdb, enable); + if (ndbenv->xdb) + rpmxdbSetFsync(ndbenv->xdb, enable); + } } static int indexSync(rpmpkgdb pkgdb, rpmxdb xdb) @@ -383,15 +431,15 @@ static rpmRC ndb_idxdbIter(dbiIndex dbi, } k = dbc->listdata + dbc->list[dbc->ilist]; kl = dbc->list[dbc->ilist + 1]; -#if 0 - if (searchType == DBC_KEY_SEARCH) { + + if (set == NULL) { dbc->ilist += 2; dbc->key = k; dbc->keylen = kl; rc = RPMRC_OK; break; } -#endif + pkglist = 0; pkglistn = 0; rc = rpmidxGet(dbc->dbi->dbi_db, k, kl, &pkglist, &pkglistn); --- ./lib/backend/ndb/rpmidx.c.orig 2020-04-28 12:50:11.812398993 +0000 +++ ./lib/backend/ndb/rpmidx.c 2021-04-21 13:25:29.011495226 +0000 @@ -66,8 +66,6 @@ typedef struct rpmidxdb_s { rpmpkgdb pkgdb; /* master database */ - char *filename; - int fd; /* our file descriptor */ int flags; int mode; @@ -166,36 +164,13 @@ static void mapcb(rpmxdb xdb, void *data set_mapped((rpmidxdb)data, newaddr, (unsigned int)newsize); } -static int rpmidxReadHeader(rpmidxdb idxdb); - static int rpmidxMap(rpmidxdb idxdb) { - if (idxdb->xdb) { - if (rpmxdbMapBlob(idxdb->xdb, idxdb->xdbid, idxdb->rdonly ? O_RDONLY : O_RDWR, mapcb, idxdb)) - return RPMRC_FAIL; - if (idxdb->file_size < 4096) { - rpmxdbUnmapBlob(idxdb->xdb, idxdb->xdbid); - return RPMRC_FAIL; - } - } else { -#ifdef IDXDB_FILESUPPORT - struct stat stb; - size_t size; - void *mapped; - if (fstat(idxdb->fd, &stb)) - return RPMRC_FAIL; - size = stb.st_size; - if (size < 4096) - return RPMRC_FAIL; - /* round up for mmap */ - size = (size + idxdb->pagesize - 1) & ~(idxdb->pagesize - 1); - mapped = mmap(0, size, idxdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE, MAP_SHARED, idxdb->fd, 0); - if (mapped == MAP_FAILED) - return RPMRC_FAIL; - set_mapped(idxdb, mapped, (unsigned int)stb.st_size); -#else + if (rpmxdbMapBlob(idxdb->xdb, idxdb->xdbid, idxdb->rdonly ? O_RDONLY : O_RDWR, mapcb, idxdb)) + return RPMRC_FAIL; + if (idxdb->file_size < 4096) { + rpmxdbUnmapBlob(idxdb->xdb, idxdb->xdbid); return RPMRC_FAIL; -#endif } return RPMRC_OK; } @@ -204,50 +179,9 @@ static void rpmidxUnmap(rpmidxdb idxdb) { if (!idxdb->head_mapped) return; - if (idxdb->xdb) { - rpmxdbUnmapBlob(idxdb->xdb, idxdb->xdbid); - } else { -#ifdef IDXDB_FILESUPPORT - size_t size = idxdb->file_size; - /* round up for munmap */ - size = (size + idxdb->pagesize - 1) & ~(idxdb->pagesize - 1); - munmap(idxdb->head_mapped, size); - set_mapped(idxdb, 0, 0); -#else - return; -#endif - } + rpmxdbUnmapBlob(idxdb->xdb, idxdb->xdbid); } -#ifdef IDXDB_FILESUPPORT -static int rpmidxReadHeader(rpmidxdb idxdb); - -/* re-open file to get the new version */ -static int rpmidxHandleObsolete(rpmidxdb idxdb) -{ - int nfd; - struct stat stb1, stb2; - - if (fstat(idxdb->fd, &stb1)) - return RPMRC_FAIL; - nfd = open(idxdb->filename, idxdb->rdonly ? O_RDONLY : O_RDWR, 0); - if (nfd == -1) - return RPMRC_FAIL; - if (fstat(nfd, &stb2)) { - close(nfd); - return RPMRC_FAIL; - } - if (stb1.st_dev == stb2.st_dev && stb1.st_ino == stb2.st_ino) { - close(nfd); - return RPMRC_FAIL; /* opened the same obsolete file */ - } - rpmidxUnmap(idxdb); - close(idxdb->fd); - idxdb->fd = nfd; - return rpmidxReadHeader(idxdb); /* re-try with new file */ -} -#endif - static int rpmidxReadHeader(rpmidxdb idxdb) { unsigned int version; @@ -272,10 +206,6 @@ static int rpmidxReadHeader(rpmidxdb idx rpmidxUnmap(idxdb); return RPMRC_FAIL; } -#ifdef IDXDB_FILESUPPORT - if (!idxdb->xdb && le2ha(idxdb->head_mapped + IDXDB_OFFSET_OBSOLETE)) - return rpmidxHandleObsolete(idxdb); -#endif idxdb->generation = le2ha(idxdb->head_mapped + IDXDB_OFFSET_GENERATION); idxdb->nslots = le2ha(idxdb->head_mapped + IDXDB_OFFSET_NSLOTS); idxdb->usedslots = le2ha(idxdb->head_mapped + IDXDB_OFFSET_USEDSLOTS); @@ -338,23 +268,6 @@ static inline void bumpGeneration(rpmidx h2lea(idxdb->generation, idxdb->head_mapped + IDXDB_OFFSET_GENERATION); } -#ifdef IDXDB_FILESUPPORT -static int createempty(rpmidxdb idxdb, off_t off, size_t size) -{ - char buf[4096]; - memset(buf, 0, sizeof(buf)); - while (size >= 4096) { - if (pwrite(idxdb->fd, buf, 4096, off) != 4096) - return RPMRC_FAIL; - off += 4096; - size -= 4096; - } - if (size > 0 && pwrite(idxdb->fd, buf, size , off) != size) - return RPMRC_FAIL; - return RPMRC_OK; -} -#endif - /*** Key management ***/ #define MURMUR_M 0x5bd1e995 @@ -454,29 +367,8 @@ static inline int equalkey(rpmidxdb idxd static int addkeypage(rpmidxdb idxdb) { unsigned int addsize = idxdb->pagesize > IDXDB_KEY_CHUNKSIZE ? idxdb->pagesize : IDXDB_KEY_CHUNKSIZE; - if (idxdb->xdb) { - if (rpmxdbResizeBlob(idxdb->xdb, idxdb->xdbid, idxdb->file_size + addsize)) - return RPMRC_FAIL; - } else { -#ifdef IDXDB_FILESUPPORT - /* don't use ftruncate because we want to create a "backed" page */ - void *newaddr; - size_t oldsize, newsize; - if (createempty(idxdb, idxdb->file_size, addsize)) - return RPMRC_FAIL; - oldsize = idxdb->file_size; - newsize = idxdb->file_size + addsize; - /* round up for mremap */ - oldsize = (oldsize + idxdb->pagesize - 1) & ~(idxdb->pagesize - 1); - newsize = (newsize + idxdb->pagesize - 1) & ~(idxdb->pagesize - 1); - newaddr = mremap(idxdb->head_mapped, oldsize, newsize, MREMAP_MAYMOVE); - if (newaddr == MAP_FAILED) - return RPMRC_FAIL; - set_mapped(idxdb, newaddr, idxdb->file_size + addsize); -#else + if (rpmxdbResizeBlob(idxdb->xdb, idxdb->xdbid, idxdb->file_size + addsize)) return RPMRC_FAIL; -#endif - } return RPMRC_OK; } @@ -591,7 +483,7 @@ static int rpmidxRebuildInternal(rpmidxd nidxdb = &nidxdb_s; memset(nidxdb, 0, sizeof(*nidxdb)); - nidxdb->pagesize = sysconf(_SC_PAGE_SIZE); + nidxdb->pagesize = rpmxdbPagesize(idxdb->xdb); /* calculate nslots the hard way, don't trust usedslots */ nslots = 0; @@ -629,45 +521,16 @@ static int rpmidxRebuildInternal(rpmidxd nidxdb->xmask = xmask; /* create new database */ - if (idxdb->xdb) { - nidxdb->xdb = idxdb->xdb; - nidxdb->xdbtag = idxdb->xdbtag; - if (rpmxdbLookupBlob(nidxdb->xdb, &nidxdb->xdbid, idxdb->xdbtag, IDXDB_XDB_SUBTAG_REBUILD, O_CREAT|O_TRUNC)) { - return RPMRC_FAIL; - } - if (rpmxdbResizeBlob(nidxdb->xdb, nidxdb->xdbid, file_size)) { - return RPMRC_FAIL; - } - if (rpmidxMap(nidxdb)) { - return RPMRC_FAIL; - } - } else { -#ifdef IDXDB_FILESUPPORT - void *mapped; - nidxdb->filename = xmalloc(strlen(idxdb->filename) + 8); - sprintf(nidxdb->filename, "%s-XXXXXX", idxdb->filename); - nidxdb->fd = mkstemp(nidxdb->filename); - if (nidxdb->fd == -1) { - free(nidxdb->filename); - return RPMRC_FAIL; - } - if (createempty(nidxdb, 0, file_size)) { - close(nidxdb->fd); - unlink(nidxdb->filename); - free(nidxdb->filename); - return RPMRC_FAIL; - } - mapped = mmap(0, file_size, idxdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE, MAP_SHARED, nidxdb->fd, 0); - if (mapped == MAP_FAILED) { - close(nidxdb->fd); - unlink(nidxdb->filename); - free(nidxdb->filename); - return RPMRC_FAIL; - } - set_mapped(nidxdb, mapped, file_size); -#else + nidxdb->xdb = idxdb->xdb; + nidxdb->xdbtag = idxdb->xdbtag; + if (rpmxdbLookupBlob(nidxdb->xdb, &nidxdb->xdbid, idxdb->xdbtag, IDXDB_XDB_SUBTAG_REBUILD, O_CREAT|O_TRUNC)) { + return RPMRC_FAIL; + } + if (rpmxdbResizeBlob(nidxdb->xdb, nidxdb->xdbid, file_size)) { + return RPMRC_FAIL; + } + if (rpmidxMap(nidxdb)) { return RPMRC_FAIL; -#endif } /* copy all entries */ @@ -700,42 +563,14 @@ static int rpmidxRebuildInternal(rpmidxd /* shrink if we have allocated excessive key space */ xfile_size = file_size - key_size + keyend + IDXDB_KEY_CHUNKSIZE; xfile_size = (xfile_size + nidxdb->pagesize - 1) & ~(nidxdb->pagesize - 1); - if (xfile_size < file_size) { - if (nidxdb->xdb) { - rpmxdbResizeBlob(nidxdb->xdb, nidxdb->xdbid, xfile_size); - } else { - if (ftruncate(nidxdb->fd, xfile_size)) { - rpmlog(RPMLOG_WARNING, _("truncate failed: %s\n"), strerror(errno)); - } - } - } + if (xfile_size < file_size) + rpmxdbResizeBlob(nidxdb->xdb, nidxdb->xdbid, xfile_size); /* now switch over to new database */ - if (idxdb->xdb) { - rpmidxUnmap(idxdb); - if (rpmxdbRenameBlob(nidxdb->xdb, &nidxdb->xdbid, idxdb->xdbtag, IDXDB_XDB_SUBTAG)) - return RPMRC_FAIL; - idxdb->xdbid = nidxdb->xdbid; - } else { -#ifdef IDXDB_FILESUPPORT - if (rename(nidxdb->filename, idxdb->filename)) { - close(nidxdb->fd); - unlink(nidxdb->filename); - free(nidxdb->filename); - return RPMRC_FAIL; - } - if (idxdb->head_mapped) { - h2lea(1, idxdb->head_mapped + IDXDB_OFFSET_OBSOLETE); - bumpGeneration(idxdb); - rpmidxUnmap(idxdb); - } - free(nidxdb->filename); - close(idxdb->fd); - idxdb->fd = nidxdb->fd; -#else + rpmidxUnmap(idxdb); + if (rpmxdbRenameBlob(nidxdb->xdb, &nidxdb->xdbid, idxdb->xdbtag, IDXDB_XDB_SUBTAG)) return RPMRC_FAIL; -#endif - } + idxdb->xdbid = nidxdb->xdbid; if (rpmidxReadHeader(idxdb)) return RPMRC_FAIL; return RPMRC_OK; @@ -1001,26 +836,14 @@ static int rpmidxListInternal(rpmidxdb i static int rpmidxInitInternal(rpmidxdb idxdb) { - if (idxdb->xdb) { - unsigned int id; - int rc = rpmxdbLookupBlob(idxdb->xdb, &id, idxdb->xdbtag, IDXDB_XDB_SUBTAG, 0); - if (rc == RPMRC_OK && id) { - idxdb->xdbid = id; - return RPMRC_OK; /* somebody else was faster */ - } - if (rc && rc != RPMRC_NOTFOUND) - return rc; - } else { -#ifdef IDXDB_FILESUPPORT - struct stat stb; - if (stat(idxdb->filename, &stb)) - return RPMRC_FAIL; - if (stb.st_size) /* somebody else was faster */ - return rpmidxHandleObsolete(idxdb); -#else - return RPMRC_FAIL; -#endif + unsigned int id; + int rc = rpmxdbLookupBlob(idxdb->xdb, &id, idxdb->xdbtag, IDXDB_XDB_SUBTAG, 0); + if (rc == RPMRC_OK && id) { + idxdb->xdbid = id; + return RPMRC_OK; /* somebody else was faster */ } + if (rc && rc != RPMRC_NOTFOUND) + return rc; return rpmidxRebuildInternal(idxdb); } @@ -1028,18 +851,12 @@ static int rpmidxLock(rpmidxdb idxdb, in { if (excl && idxdb->rdonly) return RPMRC_FAIL; - if (idxdb->xdb) - return rpmxdbLock(idxdb->xdb, excl); - else - return rpmpkgLock(idxdb->pkgdb, excl); + return rpmxdbLock(idxdb->xdb, excl); } static int rpmidxUnlock(rpmidxdb idxdb, int excl) { - if (idxdb->xdb) - return rpmxdbUnlock(idxdb->xdb, excl); - else - return rpmpkgUnlock(idxdb->pkgdb, excl); + return rpmxdbUnlock(idxdb->xdb, excl); } static int rpmidxLockReadHeader(rpmidxdb idxdb, int excl) @@ -1065,43 +882,7 @@ static int rpmidxInit(rpmidxdb idxdb) int rpmidxOpen(rpmidxdb *idxdbp, rpmpkgdb pkgdb, const char *filename, int flags, int mode) { -#ifdef IDXDB_FILESUPPORT - struct stat stb; - rpmidxdb idxdb; - - *idxdbp = 0; - idxdb = xcalloc(1, sizeof(*idxdb)); - idxdb->filename = xstrdup(filename); - if ((flags & (O_RDONLY|O_RDWR)) == O_RDONLY) - idxdb->rdonly = 1; - if ((idxdb->fd = open(filename, flags, mode)) == -1) { - free(idxdb->filename); - free(idxdb); - return RPMRC_FAIL; - } - if (fstat(idxdb->fd, &stb)) { - close(idxdb->fd); - free(idxdb->filename); - free(idxdb); - return RPMRC_FAIL; - } - idxdb->pkgdb = pkgdb; - idxdb->flags = flags; - idxdb->mode = mode; - idxdb->pagesize = sysconf(_SC_PAGE_SIZE); - if (stb.st_size == 0) { - if (rpmidxInit(idxdb)) { - close(idxdb->fd); - free(idxdb->filename); - free(idxdb); - return RPMRC_FAIL; - } - } - *idxdbp = idxdb; - return RPMRC_OK; -#else return RPMRC_FAIL; -#endif } int rpmidxOpenXdb(rpmidxdb *idxdbp, rpmpkgdb pkgdb, rpmxdb xdb, unsigned int xdbtag) @@ -1121,14 +902,12 @@ int rpmidxOpenXdb(rpmidxdb *idxdbp, rpmp return RPMRC_FAIL; } idxdb = xcalloc(1, sizeof(*idxdb)); - idxdb->fd = -1; idxdb->xdb = xdb; idxdb->xdbtag = xdbtag; idxdb->xdbid = id; idxdb->pkgdb = pkgdb; - idxdb->pagesize = sysconf(_SC_PAGE_SIZE); - if (rpmxdbIsRdonly(xdb)) - idxdb->rdonly = 1; + idxdb->pagesize = rpmxdbPagesize(xdb); + idxdb->rdonly = rpmxdbIsRdonly(xdb) ? 1 : 0; if (!id) { if (rpmidxInit(idxdb)) { free(idxdb); @@ -1165,12 +944,6 @@ int rpmidxDelXdb(rpmpkgdb pkgdb, rpmxdb void rpmidxClose(rpmidxdb idxdb) { rpmidxUnmap(idxdb); - if (idxdb->fd >= 0) { - close(idxdb->fd); - idxdb->fd = -1; - } - if (idxdb->filename) - free(idxdb->filename); free(idxdb); } @@ -1229,11 +1002,7 @@ int rpmidxStats(rpmidxdb idxdb) if (rpmidxLockReadHeader(idxdb, 0)) return RPMRC_FAIL; printf("--- IndexDB Stats\n"); - if (idxdb->xdb) { - printf("Xdb tag: %d, id: %d\n", idxdb->xdbtag, idxdb->xdbid); - } else { - printf("Filename: %s\n", idxdb->filename); - } + printf("Xdb tag: %d, id: %d\n", idxdb->xdbtag, idxdb->xdbid); printf("Generation: %u\n", idxdb->generation); printf("Slots: %u\n", idxdb->nslots); printf("Used slots: %u\n", idxdb->usedslots); --- ./lib/backend/ndb/rpmpkg.c.orig 2020-04-28 12:50:11.813399005 +0000 +++ ./lib/backend/ndb/rpmpkg.c 2021-04-21 13:25:29.011495226 +0000 @@ -7,7 +7,6 @@ #include <sys/file.h> #include <fcntl.h> #include <stdio.h> -#include <time.h> #include <unistd.h> #include <string.h> #include <stdlib.h> @@ -19,10 +18,6 @@ #define RPMRC_NOTFOUND 1 #define RPMRC_OK 0 -#ifdef RPMPKG_LZO -static int rpmpkgLZOCompress(unsigned char **blobp, unsigned int *bloblp); -static int rpmpkgLZODecompress(unsigned char **blobp, unsigned int *bloblp); -#endif static int rpmpkgVerifyblob(rpmpkgdb pkgdb, unsigned int pkgidx, unsigned int blkoff, unsigned int blkcnt); @@ -49,23 +44,19 @@ typedef struct rpmpkgdb_s { unsigned int nextpkgidx; struct pkgslot_s *slots; - unsigned int aslots; /* allocated slots */ unsigned int nslots; /* used slots */ unsigned int *slothash; unsigned int nslothash; unsigned int freeslot; /* first free slot */ - int slotorder; + int ordered; /* slots are ordered by the blk offsets */ char *filename; unsigned int fileblks; /* file size in blks */ int dofsync; } * rpmpkgdb; -#define SLOTORDER_UNORDERED 0 -#define SLOTORDER_BLKOFF 1 - static inline unsigned int le2h(unsigned char *p) { @@ -144,10 +135,6 @@ static int rpmpkgReadHeader(rpmpkgdb pkg if (pkgdb->slots && (pkgdb->generation != generation || pkgdb->slotnpages != slotnpages)) { free(pkgdb->slots); pkgdb->slots = 0; - if (pkgdb->slothash) { - free(pkgdb->slothash); - pkgdb->slothash = 0; - } } pkgdb->generation = generation; pkgdb->slotnpages = slotnpages; @@ -156,6 +143,15 @@ static int rpmpkgReadHeader(rpmpkgdb pkg return RPMRC_OK; } +static int rpmpkgFsync(rpmpkgdb pkgdb) +{ +#ifdef HAVE_FDATASYNC + return fdatasync(pkgdb->fd); +#else + return fsync(pkgdb->fd); +#endif +} + static int rpmpkgWriteHeader(rpmpkgdb pkgdb) { unsigned char header[PKGDB_HEADER_SIZE]; @@ -168,7 +164,7 @@ static int rpmpkgWriteHeader(rpmpkgdb pk if (pwrite(pkgdb->fd, header, sizeof(header), 0) != sizeof(header)) { return RPMRC_FAIL; } - if (pkgdb->dofsync && fsync(pkgdb->fd)) + if (pkgdb->dofsync && rpmpkgFsync(pkgdb)) return RPMRC_FAIL; /* write error */ return RPMRC_OK; } @@ -191,6 +187,7 @@ static inline unsigned int hashpkgidx(un return h; } +/* (re-)create a hash mapping pkgidx numbers to slots */ static int rpmpkgHashSlots(rpmpkgdb pkgdb) { unsigned int nslots, num; @@ -199,14 +196,14 @@ static int rpmpkgHashSlots(rpmpkgdb pkgd int i; pkgslot *slot; - pkgdb->nslothash = 0; - num = pkgdb->nslots; + num = pkgdb->nslots + 32; while (num & (num - 1)) num = num & (num - 1); num *= 4; hash = pkgdb->slothash; if (!hash || pkgdb->nslothash != num) { - free(pkgdb->slothash); + if (hash) + free(hash); hash = pkgdb->slothash = xcalloc(num, sizeof(unsigned int)); pkgdb->nslothash = num; } else { @@ -219,8 +216,6 @@ static int rpmpkgHashSlots(rpmpkgdb pkgd ; hash[h] = i + 1; } - pkgdb->slothash = hash; - pkgdb->nslothash = num; return RPMRC_OK; } @@ -238,10 +233,6 @@ static int rpmpkgReadSlots(rpmpkgdb pkgd free(pkgdb->slots); pkgdb->slots = 0; } - if (pkgdb->slothash) { - free(pkgdb->slothash); - pkgdb->slothash = 0; - } pkgdb->nslots = 0; pkgdb->freeslot = 0; @@ -253,8 +244,7 @@ static int rpmpkgReadSlots(rpmpkgdb pkgd fileblks = stb.st_size / BLK_SIZE; /* read (and somewhat verify) all slots */ - pkgdb->aslots = slotnpages * (PAGE_SIZE / SLOT_SIZE); - pkgdb->slots = xcalloc(pkgdb->aslots, sizeof(*pkgdb->slots)); + pkgdb->slots = xcalloc(slotnpages * (PAGE_SIZE / SLOT_SIZE), sizeof(*pkgdb->slots)); i = 0; slot = pkgdb->slots; minblkoff = slotnpages * (PAGE_SIZE / BLK_SIZE); @@ -290,7 +280,7 @@ static int rpmpkgReadSlots(rpmpkgdb pkgd } } pkgdb->nslots = i; - pkgdb->slotorder = SLOTORDER_UNORDERED; /* XXX: always order? */ + pkgdb->ordered = 0; pkgdb->fileblks = fileblks; pkgdb->freeslot = freeslot; if (rpmpkgHashSlots(pkgdb)) { @@ -308,15 +298,13 @@ static int orderslots_blkoff_cmp(const v return blkoffa > blkoffb ? 1 : blkoffa < blkoffb ? -1 : 0; } -static void rpmpkgOrderSlots(rpmpkgdb pkgdb, int slotorder) +static void rpmpkgOrderSlots(rpmpkgdb pkgdb) { - if (pkgdb->slotorder == slotorder) + if (pkgdb->ordered) return; - if (slotorder == SLOTORDER_BLKOFF) { - if (pkgdb->nslots > 1) - qsort(pkgdb->slots, pkgdb->nslots, sizeof(*pkgdb->slots), orderslots_blkoff_cmp); - } - pkgdb->slotorder = slotorder; + if (pkgdb->nslots > 1) + qsort(pkgdb->slots, pkgdb->nslots, sizeof(*pkgdb->slots), orderslots_blkoff_cmp); + pkgdb->ordered = 1; rpmpkgHashSlots(pkgdb); } @@ -331,6 +319,8 @@ static inline pkgslot *rpmpkgFindSlot(rp return 0; } +/* Find an empty space for blkcnt blocks. If dontprepend is true, ignore + the space between the slot area and the first blob */ static int rpmpkgFindEmptyOffset(rpmpkgdb pkgdb, unsigned int pkgidx, unsigned int blkcnt, unsigned *blkoffp, pkgslot **oldslotp, int dontprepend) { unsigned int i, nslots = pkgdb->nslots; @@ -339,8 +329,8 @@ static int rpmpkgFindEmptyOffset(rpmpkgd unsigned int lastblkend = pkgdb->slotnpages * (PAGE_SIZE / BLK_SIZE); pkgslot *slot, *oldslot = 0; - if (pkgdb->slotorder != SLOTORDER_BLKOFF) - rpmpkgOrderSlots(pkgdb, SLOTORDER_BLKOFF); + if (!pkgdb->ordered) + rpmpkgOrderSlots(pkgdb); if (dontprepend && nslots) { lastblkend = pkgdb->slots[0].blkoff; @@ -373,14 +363,15 @@ static int rpmpkgFindEmptyOffset(rpmpkgd return RPMRC_OK; } +/* verify the blobs to the left and right of a free area */ static int rpmpkgNeighbourCheck(rpmpkgdb pkgdb, unsigned int blkoff, unsigned int blkcnt, unsigned int *newblkcnt) { unsigned int i, nslots = pkgdb->nslots; unsigned int lastblkend = pkgdb->slotnpages * (PAGE_SIZE / BLK_SIZE); pkgslot *slot, *left = 0, *right = 0; - if (pkgdb->slotorder != SLOTORDER_BLKOFF) - rpmpkgOrderSlots(pkgdb, SLOTORDER_BLKOFF); + if (!pkgdb->ordered) + rpmpkgOrderSlots(pkgdb); if (blkoff < lastblkend) return RPMRC_FAIL; for (i = 0, slot = pkgdb->slots; i < nslots; i++, slot++) { @@ -404,7 +395,7 @@ static int rpmpkgNeighbourCheck(rpmpkgdb if (right && rpmpkgVerifyblob(pkgdb, right->pkgidx, right->blkoff, right->blkcnt) != RPMRC_OK) return RPMRC_FAIL; *newblkcnt = right ? right->blkoff - blkoff : blkcnt; - /* bounds are intect. free area. */ + /* bounds are intact. ok to zero area. */ return RPMRC_OK; } @@ -424,6 +415,7 @@ static int rpmpkgWriteslot(rpmpkgdb pkgd return RPMRC_FAIL; } pkgdb->generation++; + /* rpmpkgFsync() is done by rpmpkgWriteHeader() */ if (rpmpkgWriteHeader(pkgdb)) { return RPMRC_FAIL; } @@ -440,7 +432,7 @@ static int rpmpkgWriteEmptySlotpage(rpmp if (pwrite(pkgdb->fd, page, PAGE_SIZE - off, pageno * PAGE_SIZE + off) != PAGE_SIZE - off) { return RPMRC_FAIL; } - if (pkgdb->dofsync && fsync(pkgdb->fd)) { + if (pkgdb->dofsync && rpmpkgFsync(pkgdb)) { return RPMRC_FAIL; /* write error */ } return RPMRC_OK; @@ -518,7 +510,7 @@ static int rpmpkgValidateZero(rpmpkgdb p /*** Blob primitives ***/ -/* head: magic + pkgidx + timestamp + bloblen */ +/* head: magic + pkgidx + generation + bloblen */ /* tail: adler32 + bloblen + magic */ #define BLOBHEAD_MAGIC ('B' | 'l' << 8 | 'b' << 16 | 'S' << 24) @@ -527,10 +519,10 @@ static int rpmpkgValidateZero(rpmpkgdb p #define BLOBHEAD_SIZE (4 + 4 + 4 + 4) #define BLOBTAIL_SIZE (4 + 4 + 4) -static int rpmpkgReadBlob(rpmpkgdb pkgdb, unsigned int pkgidx, unsigned int blkoff, unsigned int blkcnt, unsigned char *blob, unsigned int *bloblp, unsigned int *tstampp) +static int rpmpkgReadBlob(rpmpkgdb pkgdb, unsigned int pkgidx, unsigned int blkoff, unsigned int blkcnt, unsigned char *blob, unsigned int *bloblp, unsigned int *generationp) { unsigned char buf[BLOBHEAD_SIZE > BLOBTAIL_SIZE ? BLOBHEAD_SIZE : BLOBTAIL_SIZE]; - unsigned int bloblen, toread, tstamp; + unsigned int bloblen, toread, generation; off_t fileoff; unsigned int adl; int verifyadler = bloblp ? 0 : 1; @@ -546,7 +538,7 @@ static int rpmpkgReadBlob(rpmpkgdb pkgdb return RPMRC_FAIL; /* bad blob */ if (le2h(buf + 4) != pkgidx) return RPMRC_FAIL; /* bad blob */ - tstamp = le2h(buf + 8); + generation = le2h(buf + 8); bloblen = le2h(buf + 12); if (blkcnt != (BLOBHEAD_SIZE + bloblen + BLOBTAIL_SIZE + BLK_SIZE - 1) / BLK_SIZE) return RPMRC_FAIL; /* bad blob */ @@ -591,8 +583,8 @@ static int rpmpkgReadBlob(rpmpkgdb pkgdb } if (bloblp) *bloblp = bloblen; - if (tstampp) - *tstampp = tstamp; + if (generationp) + *generationp = generation; return RPMRC_OK; } @@ -651,7 +643,7 @@ static int rpmpkgWriteBlob(rpmpkgdb pkgd /* update file length */ if (blkoff + blkcnt > pkgdb->fileblks) pkgdb->fileblks = blkoff + blkcnt; - if (pkgdb->dofsync && fsync(pkgdb->fd)) { + if (pkgdb->dofsync && rpmpkgFsync(pkgdb)) { return RPMRC_FAIL; /* write error */ } return RPMRC_OK; @@ -663,7 +655,7 @@ static int rpmpkgDelBlob(rpmpkgdb pkgdb, return RPMRC_FAIL; if (rpmpkgZeroBlks(pkgdb, blkoff, blkcnt)) return RPMRC_FAIL; - if (pkgdb->dofsync && fsync(pkgdb->fd)) + if (pkgdb->dofsync && rpmpkgFsync(pkgdb)) return RPMRC_FAIL; /* write error */ return RPMRC_OK; } @@ -675,14 +667,14 @@ static int rpmpkgMoveBlob(rpmpkgdb pkgdb unsigned int blkoff = slot->blkoff; unsigned int blkcnt = slot->blkcnt; unsigned char *blob; - unsigned int tstamp, blobl; + unsigned int generation, blobl; blob = xmalloc((size_t)blkcnt * BLK_SIZE); - if (rpmpkgReadBlob(pkgdb, pkgidx, blkoff, blkcnt, blob, &blobl, &tstamp)) { + if (rpmpkgReadBlob(pkgdb, pkgidx, blkoff, blkcnt, blob, &blobl, &generation)) { free(blob); return RPMRC_FAIL; } - if (rpmpkgWriteBlob(pkgdb, pkgidx, newblkoff, blkcnt, blob, blobl, tstamp)) { + if (rpmpkgWriteBlob(pkgdb, pkgidx, newblkoff, blkcnt, blob, blobl, generation)) { free(blob); return RPMRC_FAIL; } @@ -694,15 +686,15 @@ static int rpmpkgMoveBlob(rpmpkgdb pkgdb return RPMRC_FAIL; } slot->blkoff = newblkoff; - pkgdb->slotorder = SLOTORDER_UNORDERED; + pkgdb->ordered = 0; return RPMRC_OK; } static int rpmpkgAddSlotPage(rpmpkgdb pkgdb) { unsigned int cutoff; - if (pkgdb->slotorder != SLOTORDER_BLKOFF) - rpmpkgOrderSlots(pkgdb, SLOTORDER_BLKOFF); + if (!pkgdb->ordered) + rpmpkgOrderSlots(pkgdb); cutoff = (pkgdb->slotnpages + 1) * (PAGE_SIZE / BLK_SIZE); /* now move every blob before cutoff */ @@ -720,7 +712,7 @@ static int rpmpkgAddSlotPage(rpmpkgdb pk if (rpmpkgMoveBlob(pkgdb, slot, newblkoff)) { return RPMRC_FAIL; } - rpmpkgOrderSlots(pkgdb, SLOTORDER_BLKOFF); + rpmpkgOrderSlots(pkgdb); } /* make sure our new page is empty */ @@ -840,6 +832,23 @@ static int rpmpkgInit(rpmpkgdb pkgdb) return rc; } +static int rpmpkgFsyncDir(const char *filename) +{ + int rc = RPMRC_OK; + DIR *pdir; + char *filenameCopy = xstrdup(filename); + + if ((pdir = opendir(dirname(filenameCopy))) == NULL) { + free(filenameCopy); + return RPMRC_FAIL; + } + if (fsync(dirfd(pdir)) == -1) + rc = RPMRC_FAIL; + closedir(pdir); + free(filenameCopy); + return rc; +} + int rpmpkgOpen(rpmpkgdb *pkgdbp, const char *filename, int flags, int mode) { struct stat stb; @@ -855,32 +864,6 @@ int rpmpkgOpen(rpmpkgdb *pkgdbp, const c free(pkgdb); return RPMRC_FAIL; } - if (flags & O_CREAT) { - char *filenameCopy; - DIR *pdir; - - filenameCopy = xstrdup(pkgdb->filename); - - if ((pdir = opendir(dirname(filenameCopy))) == NULL) { - free(filenameCopy); - close(pkgdb->fd); - free(pkgdb->filename); - free(pkgdb); - return RPMRC_FAIL; - } - - if (fsync(dirfd(pdir)) == -1) { - closedir(pdir); - free(filenameCopy); - close(pkgdb->fd); - free(pkgdb->filename); - free(pkgdb); - return RPMRC_FAIL; - } - closedir(pdir); - free(filenameCopy); - - } if (fstat(pkgdb->fd, &stb)) { close(pkgdb->fd); free(pkgdb->filename); @@ -888,6 +871,13 @@ int rpmpkgOpen(rpmpkgdb *pkgdbp, const c return RPMRC_FAIL; } if (stb.st_size == 0) { + /* created new database */ + if (rpmpkgFsyncDir(pkgdb->filename)) { + close(pkgdb->fd); + free(pkgdb->filename); + free(pkgdb); + return RPMRC_FAIL; + } if (rpmpkgInit(pkgdb)) { close(pkgdb->fd); free(pkgdb->filename); @@ -951,6 +941,7 @@ static int rpmpkgPutInternal(rpmpkgdb pk pkgslot *oldslot; /* we always read all slots when writing, just in case */ + /* this also will set pkgdb->freeslot */ if (rpmpkgReadSlots(pkgdb)) { return RPMRC_FAIL; } @@ -974,7 +965,7 @@ static int rpmpkgPutInternal(rpmpkgdb pk return RPMRC_FAIL; } /* write new blob */ - if (rpmpkgWriteBlob(pkgdb, pkgidx, blkoff, blkcnt, blob, blobl, (unsigned int)time(0))) { + if (rpmpkgWriteBlob(pkgdb, pkgidx, blkoff, blkcnt, blob, blobl, pkgdb->generation)) { return RPMRC_FAIL; } /* write slot */ @@ -999,7 +990,7 @@ static int rpmpkgPutInternal(rpmpkgdb pk /* just update the slot, no need to free the slot data */ oldslot->blkoff = blkoff; oldslot->blkcnt = blkcnt; - pkgdb->slotorder = SLOTORDER_UNORDERED; + pkgdb->ordered = 0; } else { free(pkgdb->slots); pkgdb->slots = 0; @@ -1016,7 +1007,7 @@ static int rpmpkgDelInternal(rpmpkgdb pk if (rpmpkgReadSlots(pkgdb)) { return RPMRC_FAIL; } - rpmpkgOrderSlots(pkgdb, SLOTORDER_BLKOFF); + rpmpkgOrderSlots(pkgdb); slot = rpmpkgFindSlot(pkgdb, pkgidx); if (!slot) { return RPMRC_OK; @@ -1042,9 +1033,10 @@ static int rpmpkgDelInternal(rpmpkgdb pk } slot->blkoff = 0; slot->blkcnt = 0; + /* try to move the last two slots, the bigger one first */ slot = pkgdb->slots + pkgdb->nslots - 2; if (slot->blkcnt < slot[1].blkcnt) - slot++; /* bigger slot first */ + slot++; /* bigger slot first */ for (i = 0; i < 2; i++, slot++) { if (slot == pkgdb->slots + pkgdb->nslots) slot -= 2; @@ -1058,7 +1050,7 @@ static int rpmpkgDelInternal(rpmpkgdb pk blkoff += slot->blkcnt; blkcnt -= slot->blkcnt; } - rpmpkgOrderSlots(pkgdb, SLOTORDER_BLKOFF); + rpmpkgOrderSlots(pkgdb); } else { slot->blkoff = 0; slot->blkcnt = 0; @@ -1097,7 +1089,7 @@ static int rpmpkgListInternal(rpmpkgdb p *npkgidxlistp = pkgdb->nslots; return RPMRC_OK; } - rpmpkgOrderSlots(pkgdb, SLOTORDER_BLKOFF); + rpmpkgOrderSlots(pkgdb); nslots = pkgdb->nslots; pkgidxlist = xcalloc(nslots + 1, sizeof(unsigned int)); for (i = 0, slot = pkgdb->slots; i < nslots; i++, slot++) { @@ -1108,6 +1100,22 @@ static int rpmpkgListInternal(rpmpkgdb p return RPMRC_OK; } +static int rpmpkgVerifyInternal(rpmpkgdb pkgdb) +{ + unsigned int i, nslots; + pkgslot *slot; + + if (rpmpkgReadSlots(pkgdb)) + return RPMRC_FAIL; + rpmpkgOrderSlots(pkgdb); + nslots = pkgdb->nslots; + for (i = 0, slot = pkgdb->slots; i < nslots; i++, slot++) { + if (rpmpkgVerifyblob(pkgdb, slot->pkgidx, slot->blkoff, slot->blkcnt)) + return RPMRC_FAIL; + } + return RPMRC_OK; +} + int rpmpkgGet(rpmpkgdb pkgdb, unsigned int pkgidx, unsigned char **blobp, unsigned int *bloblp) { int rc; @@ -1120,10 +1128,6 @@ int rpmpkgGet(rpmpkgdb pkgdb, unsigned i return RPMRC_FAIL; rc = rpmpkgGetInternal(pkgdb, pkgidx, blobp, bloblp); rpmpkgUnlock(pkgdb, 0); -#ifdef RPMPKG_LZO - if (!rc) - rc = rpmpkgLZODecompress(blobp, bloblp); -#endif return rc; } @@ -1136,16 +1140,7 @@ int rpmpkgPut(rpmpkgdb pkgdb, unsigned i } if (rpmpkgLockReadHeader(pkgdb, 1)) return RPMRC_FAIL; -#ifdef RPMPKG_LZO - if (rpmpkgLZOCompress(&blob, &blobl)) { - rpmpkgUnlock(pkgdb, 1); - return RPMRC_FAIL; - } -#endif rc = rpmpkgPutInternal(pkgdb, pkgidx, blob, blobl); -#ifdef RPMPKG_LZO - free(blob); -#endif rpmpkgUnlock(pkgdb, 1); return rc; } @@ -1177,6 +1172,16 @@ int rpmpkgList(rpmpkgdb pkgdb, unsigned return rc; } +int rpmpkgVerify(rpmpkgdb pkgdb) +{ + int rc; + if (rpmpkgLockReadHeader(pkgdb, 0)) + return RPMRC_FAIL; + rc = rpmpkgVerifyInternal(pkgdb); + rpmpkgUnlock(pkgdb, 0); + return rc; +} + int rpmpkgNextPkgIdx(rpmpkgdb pkgdb, unsigned int *pkgidxp) { if (rpmpkgLockReadHeader(pkgdb, 1)) @@ -1226,64 +1231,3 @@ int rpmpkgStats(rpmpkgdb pkgdb) return RPMRC_OK; } -#ifdef RPMPKG_LZO - -#include "lzo/lzoconf.h" -#include "lzo/lzo1x.h" - -#define BLOBLZO_MAGIC ('L' | 'Z' << 8 | 'O' << 16 | 'B' << 24) - -static int rpmpkgLZOCompress(unsigned char **blobp, unsigned int *bloblp) -{ - unsigned char *blob = *blobp; - unsigned int blobl = *bloblp; - unsigned char *lzoblob, *workmem; - unsigned int lzoblobl; - lzo_uint blobl2; - - if (lzo_init() != LZO_E_OK) { - return RPMRC_FAIL; - } - workmem = xmalloc(LZO1X_1_MEM_COMPRESS); - lzoblobl = 4 + 4 + blobl + blobl / 16 + 64 + 3; - lzoblob = xmalloc(lzoblobl); - h2le(BLOBLZO_MAGIC, lzoblob); - h2le(blobl, lzoblob + 4); - if (lzo1x_1_compress(blob, blobl, lzoblob + 8, &blobl2, workmem) != LZO_E_OK) { - free(workmem); - free(lzoblob); - return RPMRC_FAIL; - } - free(workmem); - *blobp = lzoblob; - *bloblp = 8 + blobl2; - return RPMRC_OK; -} - -static int rpmpkgLZODecompress(unsigned char **blobp, unsigned int *bloblp) -{ - unsigned char *lzoblob = *blobp; - unsigned int lzoblobl = *bloblp; - unsigned char *blob; - unsigned int blobl; - lzo_uint blobl2; - - if (!lzoblob || lzoblobl < 8) - return RPMRC_FAIL; - if (le2h(lzoblob) != BLOBLZO_MAGIC) - return RPMRC_FAIL; - if (lzo_init() != LZO_E_OK) - return RPMRC_FAIL; - blobl = le2h(lzoblob + 4); - blob = xmalloc(blobl ? blobl : 1); - if (lzo1x_decompress(lzoblob + 8, lzoblobl - 8, blob, &blobl2, 0) != LZO_E_OK || blobl2 != blobl) { - free(blob); - return RPMRC_FAIL; - } - free(lzoblob); - *blobp = blob; - *bloblp = blobl; - return RPMRC_OK; -} - -#endif --- ./lib/backend/ndb/rpmpkg.h.orig 2020-04-28 12:50:11.813399005 +0000 +++ ./lib/backend/ndb/rpmpkg.h 2021-04-21 13:25:29.011495226 +0000 @@ -12,6 +12,7 @@ int rpmpkgGet(rpmpkgdb pkgdb, unsigned i int rpmpkgPut(rpmpkgdb pkgdb, unsigned int pkgidx, unsigned char *blob, unsigned int blobl); int rpmpkgDel(rpmpkgdb pkgdb, unsigned int pkgidx); int rpmpkgList(rpmpkgdb pkgdb, unsigned int **pkgidxlistp, unsigned int *npkgidxlistp); +int rpmpkgVerify(rpmpkgdb pkgdb); int rpmpkgNextPkgIdx(rpmpkgdb pkgdb, unsigned int *pkgidxp); int rpmpkgGeneration(rpmpkgdb pkgdb, unsigned int *generationp); --- ./lib/backend/ndb/rpmxdb.c.orig 2020-04-28 12:50:11.813399005 +0000 +++ ./lib/backend/ndb/rpmxdb.c 2021-04-21 13:25:29.015495219 +0000 @@ -36,6 +36,7 @@ typedef struct rpmxdb_s { unsigned int usergeneration; unsigned char *mapped; + int mapflags; unsigned int mappedlen; struct xdb_slot { @@ -95,12 +96,52 @@ static inline void h2lea(unsigned int x, #define SLOT_MAGIC ('S' | 'l' << 8 | 'o' << 16) -#define SLOT_SIZE 16 +#define SLOT_SIZE 16 #define SLOT_START (XDB_HEADER_SIZE / SLOT_SIZE) -static void rpmxdbUnmap(rpmxdb xdb) + +/* low level map/remap a file into memory */ +static void *mapmem(void *oldaddr, size_t oldsize, size_t newsize, int prot, int fd, off_t offset) { - munmap(xdb->mapped, xdb->mappedlen); + if (oldaddr) { +#if HAVE_MREMAP + return mremap(oldaddr, oldsize, newsize, MREMAP_MAYMOVE); +#else + void *mapped = mmap(0, newsize, prot, MAP_SHARED, fd, offset); + if (mapped != MAP_FAILED) + munmap(oldaddr, oldsize); + return mapped; +#endif + } + return mmap(0, newsize, prot, MAP_SHARED, fd, offset); +} + +/* unmap a mapped region */ +static void unmapmem(void *addr, size_t size) +{ + munmap(addr, size); +} + +#define ROUNDTOSYSTEMPAGE(xdb, size) (((size) + (xdb->systempagesize - 1)) & ~(xdb->systempagesize - 1)) + +/* xdb header mapping functions */ +static int mapheader(rpmxdb xdb, unsigned int slotnpages) +{ + unsigned char *mapped; + size_t mappedlen = slotnpages * xdb->pagesize; + + mappedlen = ROUNDTOSYSTEMPAGE(xdb, mappedlen); + mapped = mapmem(xdb->mapped, xdb->mappedlen, mappedlen, xdb->mapflags, xdb->fd, 0); + if ((void *)mapped == MAP_FAILED) + return RPMRC_FAIL; + xdb->mapped = mapped; + xdb->mappedlen = mappedlen; + return RPMRC_OK; +} + +static void unmapheader(rpmxdb xdb) +{ + unmapmem(xdb->mapped, xdb->mappedlen); xdb->mapped = 0; xdb->mappedlen = 0; } @@ -120,9 +161,9 @@ static int mapslot(rpmxdb xdb, struct xd shift = off & (xdb->systempagesize - 1); off -= shift; size += shift; - size = (size + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); + size = ROUNDTOSYSTEMPAGE(xdb, size); } - mapped = mmap(0, size, slot->mapflags, MAP_SHARED, xdb->fd, off); + mapped = mapmem(0, 0, size, slot->mapflags, xdb->fd, off); if (mapped == MAP_FAILED) return RPMRC_FAIL; slot->mapped = (unsigned char *)mapped + shift; @@ -131,44 +172,41 @@ static int mapslot(rpmxdb xdb, struct xd static void unmapslot(rpmxdb xdb, struct xdb_slot *slot) { - size_t size; unsigned char *mapped = slot->mapped; + size_t size; if (!mapped) return; size = slot->pagecnt * xdb->pagesize; if (xdb->pagesize != xdb->systempagesize) { size_t off = slot->startpage * xdb->pagesize; size_t shift = off & (xdb->systempagesize - 1); - mapped -= shift; size += shift; - size = (size + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); + size = ROUNDTOSYSTEMPAGE(xdb, size); + mapped -= shift; } - munmap(mapped, size); + unmapmem(mapped, size); slot->mapped = 0; } static int remapslot(rpmxdb xdb, struct xdb_slot *slot, unsigned int newpagecnt) { - void *mapped; + unsigned char *mapped = slot->mapped; size_t off, oldsize, newsize, shift; + oldsize = slot->pagecnt * xdb->pagesize; newsize = newpagecnt * xdb->pagesize; off = slot->startpage * xdb->pagesize; shift = 0; if (xdb->pagesize != xdb->systempagesize) { - off = slot->startpage * xdb->pagesize; shift = off & (xdb->systempagesize - 1); off -= shift; oldsize += shift; - oldsize = (oldsize + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); newsize += shift; - newsize = (newsize + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); + oldsize = ROUNDTOSYSTEMPAGE(xdb, oldsize); + newsize = ROUNDTOSYSTEMPAGE(xdb, newsize); } - if (slot->mapped) - mapped = mremap(slot->mapped - shift, oldsize, newsize, MREMAP_MAYMOVE); - else - mapped = mmap(0, newsize, slot->mapflags, MAP_SHARED, xdb->fd, off); - if (mapped == MAP_FAILED) + mapped = mapmem(mapped ? mapped - shift : 0, oldsize, newsize, slot->mapflags, xdb->fd, off); + if ((void *)mapped == MAP_FAILED) return RPMRC_FAIL; slot->mapped = (unsigned char *)mapped + shift; slot->pagecnt = newpagecnt; @@ -186,11 +224,33 @@ static int usedslots_cmp(const void *a, return sa->startpage > sb->startpage ? 1 : -1; } +static int rpmxdbReadHeaderRaw(rpmxdb xdb, unsigned int *generationp, unsigned int *slotnpagesp, unsigned int *pagesizep, unsigned int *usergenerationp) +{ + unsigned int header[XDB_HEADER_SIZE / sizeof(unsigned int)]; + unsigned int version; + if (pread(xdb->fd, header, sizeof(header), 0) != sizeof(header)) + return RPMRC_FAIL; + if (le2ha((unsigned char *)header + XDB_OFFSET_MAGIC) != XDB_MAGIC) + return RPMRC_FAIL; + version = le2ha((unsigned char *)header + XDB_OFFSET_VERSION); + if (version != XDB_VERSION) { + rpmlog(RPMLOG_ERR, _("rpmxdb: Version mismatch. Expected version: %u. " + "Found version: %u\n"), XDB_VERSION, version); + return RPMRC_FAIL; + } + *generationp = le2ha((unsigned char *)header + XDB_OFFSET_GENERATION); + *slotnpagesp = le2ha((unsigned char *)header + XDB_OFFSET_SLOTNPAGES); + *pagesizep = le2ha((unsigned char *)header + XDB_OFFSET_PAGESIZE); + *usergenerationp = le2ha((unsigned char *)header + XDB_OFFSET_USERGENERATION); + if (!*slotnpagesp || !*pagesizep) + return RPMRC_FAIL; + return RPMRC_OK; +} + static int rpmxdbReadHeader(rpmxdb xdb) { struct xdb_slot *slot; - unsigned int header[XDB_HEADER_SIZE / sizeof(unsigned int)]; - unsigned int slotnpages, pagesize, generation, usergeneration, version; + unsigned int slotnpages, pagesize, generation, usergeneration; unsigned int page, *lastfreep; unsigned char *pageptr; struct xdb_slot *slots, **usedslots, *lastslot; @@ -198,52 +258,31 @@ static int rpmxdbReadHeader(rpmxdb xdb) unsigned int usedblobpages; int i, nused, slotno; struct stat stb; - size_t mapsize; if (xdb->mapped) { if (le2ha(xdb->mapped + XDB_OFFSET_GENERATION) == xdb->generation) { return RPMRC_OK; } - rpmxdbUnmap(xdb); + unmapheader(xdb); } if (fstat(xdb->fd, &stb)) { return RPMRC_FAIL; } - if (pread(xdb->fd, header, sizeof(header), 0) != sizeof(header)) { - return RPMRC_FAIL; - } - if (le2ha((unsigned char *)header + XDB_OFFSET_MAGIC) != XDB_MAGIC) - return RPMRC_FAIL; - version = le2ha((unsigned char *)header + XDB_OFFSET_VERSION); - if (version != XDB_VERSION) { - rpmlog(RPMLOG_ERR, _("rpmxdb: Version mismatch. Expected version: %u. " - "Found version: %u\n"), XDB_VERSION, version); + if (rpmxdbReadHeaderRaw(xdb, &generation, &slotnpages, &pagesize, &usergeneration)) return RPMRC_FAIL; - } - - generation = le2ha((unsigned char *)header + XDB_OFFSET_GENERATION); - slotnpages = le2ha((unsigned char *)header + XDB_OFFSET_SLOTNPAGES); - pagesize = le2ha((unsigned char *)header + XDB_OFFSET_PAGESIZE); - usergeneration = le2ha((unsigned char *)header + XDB_OFFSET_USERGENERATION); - if (!slotnpages || !pagesize || stb.st_size % pagesize != 0) + if (stb.st_size % pagesize != 0) return RPMRC_FAIL; xdb->pagesize = pagesize; + xdb->mapflags = xdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE; - /* round up */ - mapsize = slotnpages * pagesize; - mapsize = (mapsize + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); - xdb->mapped = mmap(0, mapsize, xdb->rdonly ? PROT_READ : PROT_READ | PROT_WRITE, MAP_SHARED, xdb->fd, 0); - if ((void *)xdb->mapped == MAP_FAILED) { - xdb->mapped = 0; + if (mapheader(xdb, slotnpages)) return RPMRC_FAIL; - } - xdb->mappedlen = mapsize; /* read in all slots */ xdb->firstfree = 0; nslots = slotnpages * (pagesize / SLOT_SIZE) - SLOT_START + 1; slots = xcalloc(nslots + 1, sizeof(struct xdb_slot)); - usedslots = xcalloc(nslots + 1, sizeof(int)); + usedslots = xcalloc(nslots + 1, sizeof(struct xdb_slot *)); nused = 0; slotno = 1; slot = slots + 1; @@ -258,7 +297,7 @@ static int rpmxdbReadHeader(rpmxdb xdb) if ((slot->subtag & 0x00ffffff) != SLOT_MAGIC) { free(slots); free(usedslots); - rpmxdbUnmap(xdb); + unmapheader(xdb); return RPMRC_FAIL; } slot->subtag = (slot->subtag >> 24) & 255; @@ -287,7 +326,7 @@ static int rpmxdbReadHeader(rpmxdb xdb) if (lastslot->startpage + lastslot->pagecnt > slot->startpage) { free(slots); free(usedslots); - rpmxdbUnmap(xdb); + unmapheader(xdb); return RPMRC_FAIL; } lastslot->next = slot->slotno; @@ -480,6 +519,23 @@ static int rpmxdbInit(rpmxdb xdb) return rc; } +static int rpmxdbFsyncDir(const char *filename) +{ + int rc = RPMRC_OK; + DIR *pdir; + char *filenameCopy = xstrdup(filename); + + if ((pdir = opendir(dirname(filenameCopy))) == NULL) { + free(filenameCopy); + return RPMRC_FAIL; + } + if (fsync(dirfd(pdir)) == -1) + rc = RPMRC_FAIL; + closedir(pdir); + free(filenameCopy); + return rc; +} + int rpmxdbOpen(rpmxdb *xdbp, rpmpkgdb pkgdb, const char *filename, int flags, int mode) { struct stat stb; @@ -497,31 +553,6 @@ int rpmxdbOpen(rpmxdb *xdbp, rpmpkgdb pk free(xdb); return RPMRC_FAIL; } - if (flags & O_CREAT) { - char *filenameCopy; - DIR *pdir; - - filenameCopy = xstrdup(xdb->filename); - - if ((pdir = opendir(dirname(filenameCopy))) == NULL) { - free(filenameCopy); - close(xdb->fd); - free(xdb->filename); - free(xdb); - return RPMRC_FAIL; - } - - if (fsync(dirfd(pdir)) == -1) { - closedir(pdir); - free(filenameCopy); - close(xdb->fd); - free(xdb->filename); - free(xdb); - return RPMRC_FAIL; - } - closedir(pdir); - free(filenameCopy); - } if (fstat(xdb->fd, &stb)) { close(xdb->fd); free(xdb->filename); @@ -529,6 +560,13 @@ int rpmxdbOpen(rpmxdb *xdbp, rpmpkgdb pk return RPMRC_FAIL; } if (stb.st_size == 0) { + /* created new database */ + if (rpmxdbFsyncDir(xdb->filename)) { + close(xdb->fd); + free(xdb->filename); + free(xdb); + return RPMRC_FAIL; + } if (rpmxdbInit(xdb)) { close(xdb->fd); free(xdb->filename); @@ -585,6 +623,7 @@ static int moveblobto(rpmxdb xdb, struct didmap = 0; oldpagecnt = oldslot->pagecnt; if (!oldslot->mapped && oldpagecnt) { + oldslot->mapflags = PROT_READ; if (mapslot(xdb, oldslot)) return RPMRC_FAIL; didmap = 1; @@ -706,10 +745,8 @@ static int moveblobstofront(rpmxdb xdb, /* add a single page containing empty slots */ static int addslotpage(rpmxdb xdb) { - unsigned char *newaddr; struct xdb_slot *slot; int i, spp, nslots; - size_t newmappedlen; if (xdb->firstfree) return RPMRC_FAIL; @@ -731,17 +768,12 @@ static int addslotpage(rpmxdb xdb) slot = xrealloc(xdb->slots, (nslots + 1 + spp) * sizeof(*slot)); xdb->slots = slot; - if (rpmxdbWriteEmptySlotpage(xdb, xdb->slotnpages)) { + if (rpmxdbWriteEmptySlotpage(xdb, xdb->slotnpages)) return RPMRC_FAIL; - } - /* remap slots */ - newmappedlen = xdb->slotnpages * xdb->pagesize + xdb->pagesize; - newmappedlen = (newmappedlen + xdb->systempagesize - 1) & ~(xdb->systempagesize - 1); - newaddr = mremap(xdb->mapped, xdb->mappedlen, newmappedlen, MREMAP_MAYMOVE); - if (newaddr == MAP_FAILED) + + /* remap the header */ + if (mapheader(xdb, xdb->slotnpages + 1)) return RPMRC_FAIL; - xdb->mapped = newaddr; - xdb->mappedlen = newmappedlen; /* update the header */ xdb->slotnpages++; @@ -763,13 +795,16 @@ static int addslotpage(rpmxdb xdb) *slot = xdb->slots[nslots]; slot->slotno = nslots + spp; xdb->slots[slot->prev].next = slot->slotno; + + /* we have a new slotpage */ xdb->nslots += spp; + xdb->slots[0].pagecnt++; /* add new free slots to the firstfree chain */ memset(xdb->slots + nslots, 0, sizeof(*slot) * spp); for (i = 0; i < spp - 1; i++) { xdb->slots[nslots + i].slotno = nslots + i; - xdb->slots[nslots + i].next = i + 1; + xdb->slots[nslots + i].next = nslots + i + 1; } xdb->slots[nslots + i].slotno = nslots + i; xdb->firstfree = nslots; @@ -896,6 +931,45 @@ int rpmxdbDelBlob(rpmxdb xdb, unsigned i return RPMRC_OK; } +int rpmxdbDelAllBlobs(rpmxdb xdb) +{ + unsigned int slotnpages, pagesize, generation, usergeneration; + if (rpmxdbLockOnly(xdb, 1)) + return RPMRC_FAIL; + /* unmap all blobs */ + if (xdb->slots) { + int i; + struct xdb_slot *slot; + for (i = 1, slot = xdb->slots + i; i < xdb->nslots; i++, slot++) { + if (slot->startpage && slot->mapped) { + unmapslot(xdb, slot); + slot->mapcallback(xdb, slot->mapcallbackdata, 0, 0); + } + } + free(xdb->slots); + xdb->slots = 0; + } + if (xdb->mapped) + unmapheader(xdb); + if (rpmxdbReadHeaderRaw(xdb, &generation, &slotnpages, &pagesize, &usergeneration)) { + rpmxdbUnlock(xdb, 1); + return RPMRC_FAIL; + } + xdb->generation = generation + 1; + xdb->slotnpages = 1; + xdb->pagesize = pagesize; + xdb->usergeneration = usergeneration; + if (rpmxdbWriteEmptySlotpage(xdb, 0)) { + rpmxdbUnlock(xdb, 1); + return RPMRC_FAIL; + } + if (ftruncate(xdb->fd, xdb->pagesize)) { + /* ftruncate failed, but that is not a problem */ + } + rpmxdbUnlock(xdb, 1); + return RPMRC_OK; +} + int rpmxdbResizeBlob(rpmxdb xdb, unsigned int id, size_t newsize) { struct xdb_slot *slot; @@ -1128,12 +1202,26 @@ int rpmxdbIsRdonly(rpmxdb xdb) return xdb->rdonly; } +unsigned int rpmxdbPagesize(rpmxdb xdb) +{ + return xdb->pagesize; +} + +static int rpmxdbFsync(rpmxdb xdb) +{ +#ifdef HAVE_FDATASYNC + return fdatasync(xdb->fd); +#else + return fsync(xdb->fd); +#endif +} + int rpmxdbSetUserGeneration(rpmxdb xdb, unsigned int usergeneration) { if (rpmxdbLockReadHeader(xdb, 1)) return RPMRC_FAIL; /* sync before the update */ - if (xdb->dofsync && fsync(xdb->fd)) { + if (xdb->dofsync && rpmxdbFsync(xdb)) { rpmxdbUnlock(xdb, 1); return RPMRC_FAIL; } --- ./lib/backend/ndb/rpmxdb.h.orig 2020-04-28 12:50:11.813399005 +0000 +++ ./lib/backend/ndb/rpmxdb.h 2021-04-21 13:25:29.015495219 +0000 @@ -7,12 +7,14 @@ int rpmxdbOpen(rpmxdb *xdbp, rpmpkgdb pk void rpmxdbClose(rpmxdb xdb); void rpmxdbSetFsync(rpmxdb xdb, int dofsync); int rpmxdbIsRdonly(rpmxdb xdb); +unsigned int rpmxdbPagesize(rpmxdb xdb); int rpmxdbLock(rpmxdb xdb, int excl); int rpmxdbUnlock(rpmxdb xdb, int excl); int rpmxdbLookupBlob(rpmxdb xdb, unsigned int *idp, unsigned int blobtag, unsigned int subtag, int flags); int rpmxdbDelBlob(rpmxdb xdb, unsigned int id) ; +int rpmxdbDelAllBlobs(rpmxdb xdb); int rpmxdbMapBlob(rpmxdb xdb, unsigned int id, int flags, void (*mapcallback)(rpmxdb xdb, void *data, void *newaddr, size_t newsize), void *mapcallbackdata); int rpmxdbUnmapBlob(rpmxdb xdb, unsigned int id);
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
