7 #if defined(WITH_DBSQL)
9 #elif defined(WITH_SQLITE)
13 extern const char *sqlite3_errmsg(sqlite3 *db)
15 extern int sqlite3_open(
20 extern int sqlite3_exec(
23 int (*callback)(
void*,
int,
char**,
char**),
28 extern int sqlite3_prepare(
32 sqlite3_stmt **ppStmt,
36 extern int sqlite3_reset(sqlite3_stmt *pStmt)
38 extern int sqlite3_step(sqlite3_stmt *pStmt)
40 extern int sqlite3_finalize( sqlite3_stmt *pStmt)
42 extern int sqlite3_close(sqlite3 * db)
55 #define _RPMREPO_INTERNAL
68 #define REPODBG(_l) if (_rpmrepo_debug) fprintf _l
74 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
75 "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"0\">\n";
81 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
82 "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"0\">\n";
88 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
89 "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"0\">\n";
95 <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
96 <repomd xmlns=\"http://linux.duke.edu/metadata/repo\">\n";
103 #include "yum_primary_xml"
108 #include "yum_filelists_xml"
113 #include "yum_other_xml"
118 #include "wnh_primary_yaml"
123 #include "wnh_filelists_yaml"
128 #include "wnh_other_yaml"
133 #include "deb_Packages"
138 #include "deb_Sources"
144 "PRAGMA synchronous = \"OFF\";",
145 "pragma locking_mode = \"EXCLUSIVE\";",
146 "CREATE TABLE conflicts ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
147 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
148 "CREATE TABLE files ( pkgKey INTEGER, name TEXT, type TEXT );",
149 "CREATE TABLE obsoletes ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
150 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT, name TEXT, arch TEXT, version TEXT, epoch TEXT, release TEXT, summary TEXT, description TEXT, url TEXT, time_file INTEGER, time_build INTEGER, rpm_license TEXT, rpm_vendor TEXT, rpm_group TEXT, rpm_buildhost TEXT, rpm_sourcerpm TEXT, rpm_header_start INTEGER, rpm_header_end INTEGER, rpm_packager TEXT, size_package INTEGER, size_installed INTEGER, size_archive INTEGER, location_href TEXT, location_base TEXT, checksum_type TEXT);",
151 "CREATE TABLE provides ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
152 "CREATE TABLE requires ( pkgKey INTEGER, name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT );",
153 "CREATE INDEX filenames ON files (name);",
154 "CREATE INDEX packageId ON packages (pkgId);",
155 "CREATE INDEX packagename ON packages (name);",
156 "CREATE INDEX pkgconflicts on conflicts (pkgKey);",
157 "CREATE INDEX pkgobsoletes on obsoletes (pkgKey);",
158 "CREATE INDEX pkgprovides on provides (pkgKey);",
159 "CREATE INDEX pkgrequires on requires (pkgKey);",
160 "CREATE INDEX providesname ON provides (name);",
161 "CREATE INDEX requiresname ON requires (name);",
162 "CREATE TRIGGER removals AFTER DELETE ON packages\
164 \n DELETE FROM files WHERE pkgKey = old.pkgKey;\
165 \n DELETE FROM requires WHERE pkgKey = old.pkgKey;\
166 \n DELETE FROM provides WHERE pkgKey = old.pkgKey;\
167 \n DELETE FROM conflicts WHERE pkgKey = old.pkgKey;\
168 \n DELETE FROM obsoletes WHERE pkgKey = old.pkgKey;\
170 "INSERT into db_info values (9, 'direct_create');",
177 "PRAGMA synchronous = \"OFF\";",
178 "pragma locking_mode = \"EXCLUSIVE\";",
179 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
180 "CREATE TABLE filelist ( pkgKey INTEGER, name TEXT, type TEXT );",
181 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);",
182 "CREATE INDEX filelistnames ON filelist (name);",
183 "CREATE INDEX keyfile ON filelist (pkgKey);",
184 "CREATE INDEX pkgId ON packages (pkgId);",
185 "CREATE TRIGGER remove_filelist AFTER DELETE ON packages\
187 \n DELETE FROM filelist WHERE pkgKey = old.pkgKey;\
189 "INSERT into db_info values (9, 'direct_create');",
196 "PRAGMA synchronous = \"OFF\";",
197 "pragma locking_mode = \"EXCLUSIVE\";",
198 "CREATE TABLE changelog ( pkgKey INTEGER, author TEXT, date INTEGER, changelog TEXT);",
199 "CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);",
200 "CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);",
201 "CREATE INDEX keychange ON changelog (pkgKey);",
202 "CREATE INDEX pkgId ON packages (pkgId);",
203 "CREATE TRIGGER remove_changelogs AFTER DELETE ON packages\
205 \n DELETE FROM changelog WHERE pkgKey = old.pkgKey;\
207 "INSERT into db_info values (9, 'direct_create');",
269 #include "yum_primary_sqlite"
280 #include "yum_filelists_sqlite"
292 #include "yum_other_sqlite"
299 .flags = REPO_FLAGS_PRETTY,
301 .tempdir =
".repodata",
302 .finaldir =
"repodata",
303 .olddir =
".olddata",
320 .Packages_init=
NULL,
321 .Packages_qfmt=
NULL,
322 .Packages_fini=
NULL,
340 .Packages_init=
NULL,
341 .Packages_qfmt=
NULL,
342 .Packages_fini=
NULL,
360 .Packages_init=
NULL,
361 .Packages_qfmt=
NULL,
362 .Packages_fini=
NULL,
380 .Packages_init=
NULL,
382 .Packages_fini=
NULL,
404 return (
Stat(fn, st) == 0);
420 stctime = sb.st_ctime;
434 (
void) vfprintf(stderr, fmt, ap);
449 const char *
type,
int compress)
453 return rpmGetPath(repo->outputdir,
"/", dir,
"/", type,
454 (repo->markup !=
NULL ? repo->markup :
""),
455 (repo->suffix !=
NULL && compress ? repo->suffix :
""),
NULL);
466 const char * item,
int current,
int total)
470 static size_t ncols = 80 - 1;
471 const char * bn = (item !=
NULL ? strrchr(item,
'/') :
NULL);
480 nb +=
fprintf(stdout,
" - %s", bn);
483 fprintf(stdout,
"%*s", (
int)(ncols - nb),
"");
485 (
void) fflush(stdout);
508 rc = (
Mkdir(dnurl, 0755) == 0 ||
errno == EEXIST ? 0 : -1);
511 dnurl =
_free(dnurl);
532 const char ** directories = repo->directories;
533 struct stat sb, *
st = &sb;
540 if (directories !=
NULL)
541 while ((dn = *directories++) !=
NULL) {
542 if (!
rpmioExists(dn, st) || !S_ISDIR(st->st_mode)) {
550 rpmrepoError(0,
_(
"Directory %s does not exist."), repo->outputdir);
554 rpmrepoError(0,
_(
"Directory %s must be writable."), repo->outputdir);
564 rpmrepoError(0,
_(
"Old data directory exists, please remove: %s"), dn);
570 static const char * dirs[] = {
".repodata",
"repodata",
NULL };
572 static const char * types[] =
573 {
"primary",
"filelists",
"other",
"repomd",
NULL };
574 const char ** dirp, ** typep;
575 for (dirp = dirs; *dirp !=
NULL; dirp++) {
576 for (typep = types; *typep !=
NULL; typep++) {
583 if (REPO_ISSET(CHECKTS) && st->st_ctime > repo->mdtimestamp)
584 repo->mdtimestamp = st->st_ctime;
592 if (repo->groupfile !=
NULL) {
593 if (REPO_ISSET(SPLIT) || repo->groupfile[0] !=
'/') {
595 repo->groupfile =
_free(repo->groupfile);
596 repo->groupfile =
fn;
600 rpmrepoError(0,
_(
"groupfile %s cannot be found."), repo->groupfile);
617 size_t flen = strlen(fn);
618 size_t slen = strlen(suffix);
619 return (flen > slen && !
strcmp(fn + flen - slen, suffix));
625 const char ** pkglist =
NULL;
635 const char * fts_name = p->
fts_name;
639 if (p->
fts_level == 0 && fts_namelen == 0) {
641 fts_namelen =
sizeof(
".") - 1;
661 if (REPO_ISSET(NOFOLLOW))
685 if (REPO_ISSET(CHECKTS)) {
688 if (repo->pkglist !=
NULL)
689 for (pkg = repo->pkglist; *pkg !=
NULL ; pkg++) {
690 struct stat sb, *
st = &sb;
694 }
else if (st->st_ctime > repo->mdtimestamp)
713 size_t nspew = (spew !=
NULL ? strlen(spew) : 0);
715 size_t nb = (nspew > 0 ?
Fwrite(spew, 1, nspew, rfile->fd) : 0);
719 rpmrepoError(0,
_(
"Fwrite failed: expected write %u != %u bytes: %s\n"),
720 (
unsigned)nspew, (
unsigned)nb,
Fstrerror(rfile->fd));
740 rpmts ts = repo->_ts;
763 const char * spew = rfile->xml_init;
764 size_t nspew = strlen(spew);
770 rfile->fd =
Fopen(fn, repo->wmode);
776 if ((tail = strstr(spew,
" packages=\"0\">\n")) !=
NULL)
777 nspew -= strlen(tail);
779 nb =
Fwrite(spew, 1, nspew, rfile->fd);
783 size_t tnb =
snprintf(buf,
sizeof(buf),
" packages=\"%d\">\n",
786 nb +=
Fwrite(buf, 1, tnb, rfile->fd);
789 rpmrepoError(0,
_(
"Fwrite failed: expected write %u != %u bytes: %s\n"),
790 (
unsigned)nspew, (
unsigned)nb,
Fstrerror(rfile->fd));
796 #if defined(WITH_SQLITE)
797 if (REPO_ISSET(DATABASE)) {
800 fn =
rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
801 rfile->type,
".sqlite",
NULL);
802 if ((xx = sqlite3_open(fn, &rfile->sqldb)) != SQLITE_OK)
803 rpmrepoError(1,
"sqlite3_open(%s): %s", fn, sqlite3_errmsg(rfile->sqldb));
804 for (stmt = rfile->sql_init; *stmt !=
NULL; stmt++) {
806 xx = sqlite3_exec(rfile->sqldb, *stmt,
NULL,
NULL, &msg);
808 rpmrepoError(1,
"sqlite3_exec(%s, \"%s\"): %s\n", fn, *stmt,
809 (msg !=
NULL ? msg :
"failed"));
818 #if defined(WITH_SQLITE)
824 static int rpmrfileSQL(
rpmrfile rfile,
const char *
msg,
int rc)
828 if (rc != SQLITE_OK || _rpmrepo_debug)
830 sqlite3_errmsg(rfile->sqldb));
839 static int rpmrfileSQLStep(
rpmrfile rfile, sqlite3_stmt * stmt)
849 rc = sqlite3_step(stmt);
852 rc = rpmrfileSQL(rfile,
"step", rc);
861 xx = rpmrfileSQL(rfile,
"reset",
862 sqlite3_reset(stmt));
873 static int rpmrfileSQLWrite(
rpmrfile rfile,
const char * cmd)
881 xx = rpmrfileSQL(rfile,
"prepare",
882 sqlite3_prepare(rfile->sqldb, cmd, (
int)strlen(cmd), &stmt, &tail));
884 xx = rpmrfileSQL(rfile,
"reset",
885 sqlite3_reset(stmt));
887 xx = rpmrfileSQLStep(rfile, stmt);
889 xx = rpmrfileSQL(rfile,
"finalize",
890 sqlite3_finalize(stmt));
903 const char ** digestp)
906 static int asAscii = 1;
907 struct stat sb, *
st = &sb;
915 memset(st, 0,
sizeof(*st));
918 fd =
Fopen(fn,
"r.ufdio");
925 #if defined(HAVE_MMAP)
926 {
void * mapped = (
void *)-1;
929 mapped = mmap(
NULL, st->st_size, PROT_READ, MAP_SHARED,
Fileno(fd), 0);
930 if (mapped != (
void *)-1) {
932 rpmts ts = repo->_ts;
942 xx = munmap(mapped, st->st_size);
948 {
char buf[64 * BUFSIZ];
953 while ((nb =
Fread(buf,
sizeof(buf[0]),
sizeof(buf), fd)) > 0)
980 static int asAscii = 1;
999 rfile->Zdigest =
NULL;
1002 #if defined(WITH_SQLITE)
1003 if (REPO_ISSET(DATABASE) && rfile->sqldb !=
NULL) {
1004 const char *dbfn =
rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1005 rfile->type,
".sqlite",
NULL);
1007 if ((xx = sqlite3_close(rfile->sqldb)) != SQLITE_OK)
1008 rpmrepoError(1,
"sqlite3_close(%s): %s", dbfn, sqlite3_errmsg(rfile->sqldb));
1009 rfile->sqldb =
NULL;
1015 xmlfn =
_free(xmlfn);
1023 algo2tagname(uint32_t algo)
1051 default: tagname =
NULL;
break;
1065 const char * spewalgo = algo2tagname(repo->algo);
1070 <data type=\"", rfile->type,
"\">\n\
1071 <checksum type=\"", spewalgo,
"\">", rfile->Zdigest,
"</checksum>\n\
1072 <timestamp>", spewtime,
"</timestamp>\n\
1073 <open-checksum type=\"",spewalgo,
"\">", rfile->digest,
"</open-checksum>\n\
1074 <location href=\"", repo->finaldir,
"/", rfile->type, (repo->markup !=
NULL ? repo->markup :
""), (repo->suffix !=
NULL ? repo->suffix :
""),
"\"/>\n\
1084 if ((rfile->fd =
Fopen(fn,
"w.ufdio")) !=
NULL) {
1099 def doRepoMetadata(
self):
1100 """wrapper to generate the repomd.xml file that stores the info on the other files"""
1101 const char * repopath =
1103 repodoc = libxml2.newDoc(
"1.0")
1104 reporoot = repodoc.newChild(None,
"repomd", None)
1105 repons = reporoot.newNs(
"http://linux.duke.edu/metadata/repo", None)
1106 reporoot.setNs(repons)
1107 repopath =
rpmGetPath(repo->outputdir,
"/", repo->tempdir,
NULL);
1110 repoid =
"garbageid";
1112 if (REPO_ISSET(DATABASE)) {
1113 if (!repo->quiet)
rpmrepoError(0,
_(
"Generating sqlite DBs"));
1115 dbversion = str(sqlitecachec.DBVERSION)
1116 except AttributeError:
1118 rp = sqlitecachec.RepodataParserSqlite(repopath, repoid, None)
1121 {
static const char * types[] =
1122 {
"primary",
"filelists",
"other",
NULL };
1123 const char ** typep;
1124 for (typep = types; *typep !=
NULL; typep++) {
1127 zfo = _gzipOpen(complete_path)
1128 uncsum = misc.checksum(algo2tagname(repo->algo), zfo)
1130 csum = misc.checksum(algo2tagname(repo->algo), complete_path)
1132 timestamp = os.stat(complete_path)[8]
1135 db_compressed_sums = {}
1137 if (REPO_ISSET(repo)) {
1138 if (repo->verbose) {
1139 time_t now = time(
NULL);
1141 *typep, ctime(&now));
1144 if (!
strcmp(*typep,
"primary"))
1145 rp.getPrimary(complete_path, csum)
1146 else if (!
strcmp(*typep,
"filelists"));
1147 rp.getFilelists(complete_path, csum)
1148 else if (!
strcmp(*typep,
"other"))
1149 rp.getOtherdata(complete_path, csum)
1151 {
const char * tmp_result_path =
1152 rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1153 *typep,
".xml.gz.sqlite",
NULL);
1154 const char * resultpath =
1155 rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1156 *typep,
".sqlite",
NULL);
1159 xx =
Rename(tmp_result_path, resultpath);
1160 tmp_result_path =
_free(tmp_result_path);
1162 rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1163 *typep,
".sqlite.bz2",
NULL);
1164 db_csums[*typep] = misc.checksum(algo2tagname(repo->algo), resultpath)
1167 bzipFile(resultpath, result_compressed)
1169 db_compressed_sums[*typep] = misc.checksum(algo2tagname(repo->algo), result_compressed)
1172 resultpath =
_free(resultpath);
1175 if (REPO_ISSET(UNIQUEMDFN)) {
1176 const char * csum_result_compressed =
1177 rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/",
1178 db_compressed_sums[*typep],
"-", *typep,
".sqlite.bz2",
NULL);
1179 xx =
Rename(result_compressed, csum_result_compressed);
1180 result_compressed =
_free(result_compressed);
1181 result_compressed = csum_result_compressed;
1186 db_timestamp = os.stat(result_compressed)[8]
1190 data = reporoot.newChild(None,
'data', None)
1191 data.newProp(
'type', db_data_type)
1192 location = data.newChild(None,
'location', None)
1193 if (repo->baseurl !=
NULL) {
1194 location.newProp(
'xml:base', repo->baseurl)
1197 location.newProp(
'href',
rpmGetPath(repo->finaldir,
"/", *typep,
".sqlite.bz2",
NULL));
1198 checksum = data.newChild(None,
'checksum', db_compressed_sums[*typep])
1199 checksum.newProp(
'type', algo2tagname(repo->algo))
1200 db_tstamp = data.newChild(None,
'timestamp', str(db_timestamp))
1201 unchecksum = data.newChild(None,
'open-checksum', db_csums[*typep])
1202 unchecksum.newProp(
'type', algo2tagname(repo->algo))
1203 database_version = data.newChild(None,
'database_version', dbversion)
1204 if (repo->verbose) {
1205 time_t now = time(
NULL);
1207 *typep, ctime(&now));
1211 data = reporoot.newChild(None,
'data', None)
1212 data.newProp(
'type', *typep)
1214 checksum = data.newChild(None,
'checksum', csum)
1215 checksum.newProp(
'type', algo2tagname(repo->algo))
1216 timestamp = data.newChild(None,
'timestamp', str(timestamp))
1217 unchecksum = data.newChild(None,
'open-checksum', uncsum)
1218 unchecksum.newProp(
'type', algo2tagname(repo->algo))
1219 location = data.newChild(None,
'location', None)
1220 if (repo->baseurl !=
NULL)
1221 location.newProp(
'xml:base', repo->baseurl)
1222 if (REPO_ISSET(UNIQUEMDFN)) {
1225 (repo->markup ? repo->markup :
""),
1226 (repo->suffix &&
strcmp(*typep,
"repomd") ? repo->suffix :
""),
NULL);
1227 dest_file =
rpmGetPath(repo->outputdir,
"/", repo->tempdir,
"/", res_file,
NULL);
1228 xx =
Rename(orig_file, dest_file);
1232 (repo->markup ? repo->markup :
""),
1233 (repo->suffix &&
strcmp(*typep,
"repomd") ? repo->suffix :
""),
NULL);
1235 location.newProp(
'href',
rpmGetPath(repo->finaldir,
"/", res_file,
NULL));
1239 if (!repo->quiet && REPO_ISSET(DATABASE))
1242 if (repo->groupfile !=
NULL) {
1243 self.addArbitraryMetadata(repo->groupfile,
'group_gz', reporoot)
1244 self.addArbitraryMetadata(repo->groupfile,
'group', reporoot, compress=False)
1249 repodoc.saveFormatFileEnc(fn,
'UTF-8', 1)
1253 (repo->markup ? repo->markup :
""),
1254 (repo->suffix &&
strcmp(*typep,
"repomd") ? repo->suffix :
""),
1266 char * output_final_dir =
1268 char * output_old_dir =
1270 struct stat sb, *
st = &sb;
1274 if ((xx =
Rename(output_final_dir, output_old_dir)) != 0)
1276 output_final_dir, output_old_dir);
1279 {
const char * output_temp_dir =
1281 if ((xx =
Rename(output_temp_dir, output_final_dir)) != 0) {
1282 xx =
Rename(output_old_dir, output_final_dir);
1283 rpmrepoError(1,
_(
"Error moving final metadata into place"));
1285 output_temp_dir =
_free(output_temp_dir);
1290 char *
const _av[] = { output_old_dir,
NULL };
1301 const char * nfn =
NULL;
1307 if ((xx =
Rmdir(opath)) != 0)
1308 rpmrepoError(1,
_(
"Could not remove old metadata directory: %s: %s"),
1309 ofn, strerror(
errno));
1314 if ((xx =
Unlink(opath)) != 0)
1315 rpmrepoError(1,
_(
"Could not remove old metadata file: %s: %s"),
1316 ofn, strerror(
errno));
1323 if ((xx =
Unlink(opath)) != 0)
1324 rpmrepoError(1,
_(
"Could not remove old metadata file: %s: %s"),
1325 ofn, strerror(
errno));
1327 if ((xx =
Rename(opath, nfn)) != 0)
1328 rpmrepoError(1,
_(
"Could not restore old non-metadata file: %s -> %s: %s"),
1329 ofn, nfn, strerror(
errno));
1336 if ((xx =
Unlink(opath)) != 0)
1337 rpmrepoError(1,
_(
"Could not remove old metadata symlink: %s: %s"),
1338 ofn, strerror(
errno));
1346 output_old_dir =
_free(output_old_dir);
1347 output_final_dir =
_free(output_final_dir);
1369 rpmts ts = repo->_ts;
1370 uint32_t algo = repo->pkgalgo;
1379 char buffer[32 * BUFSIZ];
1380 size_t nb =
sizeof(buffer);
1382 while ((nr =
Fread(buffer,
sizeof(buffer[0]), nb, fd)) ==
nb)
1385 fprintf(stderr,
_(
"%s: Fread(%s) failed: %s\n"),
1389 static int asAscii = 1;
1393 digest =
_free(digest);
1428 const char * msg =
NULL;
1436 #if defined(WITH_SQLITE)
1443 static const char * rfileHeaderSprintfHack(
Header h,
const char * qfmt)
1447 static const char mark[] =
"'XXX'";
1448 static size_t nmark =
sizeof(
"'XXX'") - 1;
1449 const char * msg =
NULL;
1460 for (f = s; *f !=
'\0' && (fe = strstr(f,
"'XXX'")) !=
NULL; fe += nmark, f =
fe)
1466 int xx =
snprintf(instance,
sizeof(instance),
"'%u'",
1468 size_t tlen = strlen(s) + nsubs * ((
int)strlen(instance) - (
int)nmark);
1474 for (f = s; *f !=
'\0' && (fe = strstr(f, mark)) !=
NULL; fe += nmark, f =
fe) {
1502 if (rfile->xml_qfmt !=
NULL) {
1507 #if defined(WITH_SQLITE)
1508 if (REPO_ISSET(DATABASE)) {
1509 if (rpmrfileSQLWrite(rfile, rfileHeaderSprintfHack(h, rfile->sql_qfmt)))
1526 const char ** pkglist = repo->pkglist;
1531 while ((pkg = *pkglist++) !=
NULL) {
1544 reldir = (pkgpath !=
NULL ? pkgpath :
rpmGetPath(repo->basedir,
"/", repo->directories[0],
NULL));
1545 self.primaryfile.write(po.do_primary_xml_dump(reldir, baseurl=repo->baseurl))
1546 self.flfile.write(po.do_filelists_xml_dump())
1547 self.otherfile.write(po.do_other_xml_dump())
1561 rpmrepoError(0,
"%d/%d - %s", repo->current, repo->pkgcount, pkg);
1576 def _getFragmentUrl(
self, url, fragment):
1578 urlparse.uses_fragment.append(
'media')
1581 (scheme, netloc,
path, query, fragid) = urlparse.urlsplit(url)
1582 return urlparse.urlunsplit((scheme, netloc,
path, query, str(fragment)))
1584 def doPkgMetadata(
self):
1585 """all the heavy lifting for the package metadata"""
1586 if (
argvCount(repo->directories) == 1) {
1587 MetaDataGenerator.doPkgMetadata(
self)
1593 for mydir in repo->directories {
1594 if (mydir[0] ==
'/')
1596 else if (mydir[0] ==
'.' && mydir[1] ==
'.' && mydir[2] ==
'/')
1601 xx =
argvAdd(&roots, thisdir);
1602 thisdir =
_free(thisdir);
1605 self.trimRpms(filematrix[mydir])
1606 repo->pkgcount =
argvCount(filematrix[mydir]);
1611 repo->baseurl =
self._getFragmentUrl(repo->baseurl, mediano)
1621 for mydir in repo->directories {
1622 repo->baseurl =
self._getFragmentUrl(repo->baseurl, mediano)
1628 repo->baseurl =
self._getFragmentUrl(repo->baseurl, 1)
1651 {
"uncompressed",
'\0', POPT_ARG_VAL, &
compression, 0,
1652 N_(
"don't compress"),
NULL },
1654 N_(
"use gzip compression"),
NULL },
1656 N_(
"use bzip2 compression"),
NULL },
1658 N_(
"use lzma compression"),
NULL },
1660 N_(
"use xz compression"),
NULL },
1667 {
"quiet",
'q', POPT_ARG_VAL, &__repo.quiet, 0,
1668 N_(
"output nothing except for serious errors"),
NULL },
1669 {
"verbose",
'v', 0,
NULL, (
int)
'v',
1670 N_(
"output more debugging info."),
NULL },
1671 {
"dryrun",
'\0', POPT_BIT_SET, &__repo.flags, REPO_FLAGS_DRYRUN,
1672 N_(
"sanity check arguments, don't create metadata"),
NULL },
1673 {
"excludes",
'x', POPT_ARG_ARGV, &__repo.exclude_patterns, 0,
1674 N_(
"glob PATTERN(s) to exclude"),
N_(
"PATTERN") },
1675 {
"includes",
'i', POPT_ARG_ARGV, &__repo.include_patterns, 0,
1676 N_(
"glob PATTERN(s) to include"),
N_(
"PATTERN") },
1677 {
"basedir",
'\0', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__repo.basedir, 0,
1678 N_(
"top level directory"),
N_(
"DIR") },
1679 {
"baseurl",
'u', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__repo.baseurl, 0,
1680 N_(
"baseurl to append on all files"),
N_(
"BASEURL") },
1682 {
"groupfile",
'g', POPT_ARG_STRING|POPT_ARGFLAG_DOC_HIDDEN, &__repo.groupfile, 0,
1683 N_(
"path to groupfile to include in metadata"),
N_(
"FILE") },
1685 {
"pretty",
'p', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__repo.flags, REPO_FLAGS_PRETTY,
1686 N_(
"make sure all xml generated is formatted"),
NULL },
1687 {
"checkts",
'C', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__repo.flags, REPO_FLAGS_CHECKTS,
1688 N_(
"check timestamps on files vs the metadata to see if we need to update"),
NULL },
1689 {
"database",
'd', POPT_BIT_SET, &__repo.flags, REPO_FLAGS_DATABASE,
1690 N_(
"create sqlite3 database files"),
NULL },
1691 {
"split",
'\0', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__repo.flags, REPO_FLAGS_SPLIT,
1692 N_(
"generate split media"),
NULL },
1693 {
"pkglist",
'l', POPT_ARG_ARGV|POPT_ARGFLAG_DOC_HIDDEN, &__repo.manifests, 0,
1694 N_(
"use only the files listed in this file from the directory specified"),
N_(
"FILE") },
1695 {
"outputdir",
'o', POPT_ARG_STRING, &__repo.outputdir, 0,
1696 N_(
"<dir> = optional directory to output to"),
N_(
"DIR") },
1697 {
"skip-symlinks",
'S', POPT_BIT_SET, &__repo.flags, REPO_FLAGS_NOFOLLOW,
1698 N_(
"ignore symlinks of packages"),
NULL },
1699 {
"unique-md-filenames",
'\0', POPT_BIT_SET|POPT_ARGFLAG_DOC_HIDDEN, &__repo.flags, REPO_FLAGS_UNIQUEMDFN,
1700 N_(
"include the file's checksum in the filename, helps with proxies"),
NULL },
1710 N_(
"Repository options:"),
NULL },
1713 N_(
"Fts(3) traversal options:"),
NULL },
1716 N_(
"Available compressions:"),
NULL },
1719 N_(
"Available digests:"),
NULL },
1722 N_(
"Common options for all rpmio executables:"),
1733 void *use = repo->_item.use;
1734 void *pool = repo->_item.pool;
1742 repo->_item.use = use;
1743 repo->_item.pool = pool;
1756 rpmrepoError(1,
"FTS_LOGICAL and FTS_PYSICAL are mutually exclusive");
1766 repo->compression = (compression >= 0 ? compression : 1);
1767 switch (repo->compression) {
1769 repo->suffix =
NULL;
1770 repo->wmode =
"w.ufdio";
1775 repo->suffix =
".gz";
1776 repo->wmode =
"w9.gzdio";
1779 repo->suffix =
".bz2";
1780 repo->wmode =
"w9.bzdio";
1783 repo->suffix =
".lzma";
1784 repo->wmode =
"w.lzdio";
1787 repo->suffix =
".xz";
1788 repo->wmode =
"w.xzdio";
1792 repo->av = poptGetArgs(repo->con);
1794 if (repo->av ==
NULL || repo->av[0] ==
NULL)
1797 if (repo->av !=
NULL)
1798 for (i = 0; repo->av[
i] !=
NULL; i++) {
1802 const char * lpath =
NULL;
1803 int ut =
urlPath(repo->av[i], &lpath);
1804 size_t nb = (size_t)(lpath - repo->av[i]);
1805 int isdir = (lpath[strlen(lpath)-1] ==
'/');
1808 if (lpath[0] !=
'/') {
1812 rpath =
_free(rpath);
1825 assert(nb <
sizeof(fullpath));
1826 strncpy(fullpath, repo->av[i], nb);
1827 fullpath[
nb] =
'\0';
1829 lpath =
_free(lpath);
1834 lpath = (isdir || (!
Stat(rpath, &sb) && S_ISDIR(sb.st_mode))
1836 if (lpath !=
NULL) {
1838 xx =
argvAdd(&repo->directories, lpath);
1839 lpath =
_free(lpath);
1841 xx =
argvAdd(&repo->pkglist, rpath);
1843 rpath =
_free(rpath);
1855 repo->primary.digest =
_free(repo->primary.digest);
1856 repo->primary.Zdigest =
_free(repo->primary.Zdigest);
1857 repo->filelists.digest =
_free(repo->filelists.digest);
1858 repo->filelists.Zdigest =
_free(repo->filelists.Zdigest);
1859 repo->other.digest =
_free(repo->other.digest);
1860 repo->other.Zdigest =
_free(repo->other.Zdigest);
1861 repo->repomd.digest =
_free(repo->repomd.digest);
1862 repo->repomd.Zdigest =
_free(repo->repomd.Zdigest);
1863 repo->outputdir =
_free(repo->outputdir);
1864 repo->pkglist =
argvFree(repo->pkglist);
1865 repo->directories =
argvFree(repo->directories);
1866 repo->manifests =
argvFree(repo->manifests);
1868 repo->excludeMire =
mireFreeAll(repo->excludeMire, repo->nexcludes);
1869 repo->includeMire =
mireFreeAll(repo->includeMire, repo->nincludes);
1871 repo->exclude_patterns =
argvFree(repo->exclude_patterns);
1872 repo->include_patterns =
argvFree(repo->include_patterns);
1874 repo->con = poptFreeContext(repo->con);
1887 if (_rpmrepoPool ==
NULL) {
1888 _rpmrepoPool =
rpmioNewPool(
"repo",
sizeof(*repo), -1, _rpmrepo_debug,
1893 memset(((
char *)repo)+
sizeof(repo->_item), 0,
sizeof(*repo)-
sizeof(repo->_item));
static void fdInitDigest(FD_t fd, pgpHashAlgo hashalgo, int _flags)
Attach digest to fd.
static const char * suffix[]
rpmop rpmtsOp(rpmts ts, rpmtsOpX opx)
Retrieve operation timestamp from a transaction set.
poptContext rpmioInit(int argc, char *const argv[], struct poptOption *optionsTable)
unsigned long int rpmtime_t
int rpmrepoDoFinalMove(rpmrepo repo)
Rename temporary repository to final paths.
struct poptOption rpmioDigestPoptTable[]
Digest options using popt.
static const char * primary_sql_init[]
FTS * Fts_open(char *const *argv, int options, int(*compar)(const FTSENT **, const FTSENT **))
Create a handle for file hierarchy traversal.
int mireApply(miRE mire, int nmire, const char *s, size_t slen, int rc)
Apply array of patterns to a string.
static struct rpmrepo_s __repo
static const char Packages_qfmt[]
void rpmrepoError(int lvl, const char *fmt,...)
Print an error message and exit (if requested).
rpmtime_t rpmswAdd(rpmop to, rpmop from)
Sum statistic counters.
void * mireFreeAll(miRE mire, int nmire)
Destroy compiled patterns.
struct rpmrfile_s * rpmrfile
static const char filelists_xml_qfmt[]
static PyObject *int type
static const char primary_yaml_qfmt[]
size_t Fwrite(const void *buf, size_t size, size_t nmemb, FD_t fd)
fwrite(3) clone.
char * xstrdup(const char *str)
FD_t Fopen(const char *path, const char *_fmode)
fopen(3) clone.
rpmrepo rpmrepoLink(rpmrepo repo)
Reference a repo wrapper instance.
static int chkSuffix(const char *fn, const char *suffix)
Check file name for a suffix.
char * rpmGetPath(const char *path,...)
Return (malloc'ed) expanded, canonicalized, file path.
static char *size_t nb
fgets(3) analogue that reads \ continuations.
int Access(const char *path, int amode)
access(2) clone.
struct rpmts_s * rpmts
The RPM Transaction Set.
static const char * filelists_sql_init[]
static const char * rpmrepoMDExpand(rpmrepo repo, rpmrfile rfile)
Return a repository metadata file item.
int rpmioMkpath(const char *path, mode_t mode, uid_t uid, gid_t gid)
Insure that directories in path exist, creating as needed.
struct poptOption rpmioFtsPoptTable[]
int Stat(const char *path, struct stat *st)
stat(2) clone.
const char ** rpmrepoGetFileList(rpmrepo repo, const char *roots[], const char *ext)
Walk file/directory trees, looking for files with an extension.
static int rpmrepoMkdir(rpmrepo repo, const char *dn)
Create directory path.
const char * rpmrepoRealpath(const char *lpath)
Return realpath(3) canonicalized absolute path.
pgpHashAlgo rpmioDigestHashAlgo
static struct poptOption repoCompressionPoptTable[]
static int rpmrepoRfileDigest(const rpmrepo repo, rpmrfile rfile, const char **digestp)
Compute digest of a file.
static const char * rfileHeaderSprintf(Header h, const char *qfmt)
Return header query.
static int rpmioExists(const char *fn, struct stat *st)
Return stat(2) for a file.
enum rpmRC_e rpmRC
RPM return codes.
Yet Another syslog(3) API clone.
memset(_r, 0, sizeof(*_r))
static const char filelists_xml_init[]
int rpmrepoDoRepoMetadata(rpmrepo repo)
Write repository manifest.
const char * Fstrerror(FD_t fd)
strerror(3) clone.
assert(key->size==sizeof(hdrNum))
static const char primary_sql_qfmt[]
int Rmdir(const char *path)
rmdir(2) clone.
int rpmrepoDoPkgMetadata(rpmrepo repo)
Write repository metadata files.
rpmioItem rpmioGetPool(rpmioPool pool, size_t size)
Get unused item from pool, or alloc a new item.
fprintf(stderr,"--> %s(%p,%p,%p) sig %p sigp %p\n", __FUNCTION__, dig, t, rsactx, sig, sigp)
int Rename(const char *oldpath, const char *newpath)
rename(2) clone.
struct rpmrepo_s * rpmrepo
static const char other_xml_qfmt[]
int argvCount(const ARGV_t argv)
Return no.
int Mkdir(const char *path, mode_t mode)
mkdir(2) clone.
int rpmDigestUpdate(DIGEST_CTX ctx, const void *data, size_t len)
Update context with next plain text buffer.
ARGV_t argvFree(ARGV_t argv)
Destroy an argv array.
static const char other_sql_qfmt[]
static struct poptOption rpmrepoOptionsTable[]
rpmRC rpmReadPackageFile(rpmts ts, FD_t fd, const char *fn, Header *hdrp)
Return package header from file handle, verifying digests/signatures.
The FD_t File Handle data structure.
const char * rpmGenPath(const char *urlroot, const char *urlmdir, const char *urlfile)
Merge 3 args into path, any or all of which may be a url.
int argvAdd(ARGV_t *argvp, ARGstr_t val)
Add a string to an argv array.
static time_t rpmioCtime(const char *fn)
Return stat(2) creation time of a file.
DIGEST_CTX rpmDigestInit(pgpHashAlgo hashalgo, rpmDigestFlags flags)
Initialize digest.
FTSENT * Fts_read(FTS *sp)
Return next node in the file hierarchy traversal.
char * rpmExpand(const char *arg,...)
Return (malloc'ed) concatenated macro expansion(s).
size_t Fread(void *buf, size_t size, size_t nmemb, FD_t fd)
fread(3) clone.
int Fclose(FD_t fd)
fclose(3) clone.
Cumulative statistics for an operation.
void argvPrint(const char *msg, ARGV_t argv, FILE *fp)
Print argv array elements.
static const char repomd_xml_init[]
static const char * other_sql_init[]
static const char primary_xml_init[]
static const char other_yaml_qfmt[]
static Header rpmrepoReadHeader(rpmrepo repo, const char *path)
Read a header from a repository package file, computing package file digest.
int Ferror(FD_t fd)
ferror(3) clone.
static int rpmrfileXMLWrite(rpmrfile rfile, const char *spew)
Write to a repository metadata file.
return strcmp(ame->name, bme->name)
urltype urlPath(const char *url, const char **pathp)
Return path component of URL.
static int snprintf(char *buf, int nb, const char *fmt,...)
int rpmrepoCheckTimeStamps(rpmrepo repo)
Check that repository time stamp is newer than any contained package.
static int rpmrepoFclose(rpmrepo repo, FD_t fd)
Close an I/O stream, accumulating uncompress/digest statistics.
Methods to handle package elements.
rpmioPool rpmioNewPool(const char *name, size_t size, int limit, int flags, char *(*dbg)(void *item), void(*init)(void *item), void(*fini)(void *item))
Create a memory pool.
rpmrepo rpmrepoNew(char **av, int flags)
Create and load a repo wrapper.
char * stpcpy(char *dest, const char *src)
static int rpmrepoOpenMDFile(const rpmrepo repo, rpmrfile rfile)
Open a repository metadata file.
Structures and prototypes used for an "rpmts" transaction set.
static const char filelists_yaml_qfmt[]
static const char * fdGetOPath(FD_t fd)
static const char * rpmrepoGetPath(rpmrepo repo, const char *dir, const char *type, int compress)
Return /repository/directory/component.markup.compression path.
static const char other_xml_fini[]
int Fts_close(FTS *sp)
Destroy a file hierarchy traversal handle.
int Fileno(FD_t fd)
fileno(3) clone.
static rpmrepo rpmrepoGetPool(rpmioPool pool)
static void
Print copy of spec file, filling in Group/Description/Summary from specspo.
static void rpmrepoProgress(rpmrepo repo, const char *item, int current, int total)
Display progress.
static void fdFiniDigest(FD_t fd, pgpHashAlgo hashalgo, void *datap, size_t *lenp, int asAscii)
char * buf
Parse (and execute) macro undefinition.
static int repoWriteMetadataDocs(rpmrepo repo)
Export all package metadata to repository metadata file(s).
static const char filelists_sql_qfmt[]
static int rpmrepoCloseMDFile(const rpmrepo repo, rpmrfile rfile)
Close a repository metadata file.
int
Save source and expand field into target.
static rpmop fdstat_op(FD_t fd, fdOpX opx)
static const char Sources_qfmt[]
void rpmDigestFinal(rpmDigestDup(md5ctx),&md5sum,&md5len, 0)
static void rpmrepoFini(void *_repo)
struct poptOption rpmioAllPoptTable[]
static const char other_xml_init[]
static int rpmrepoInitPopt(rpmrepo repo, char **av)
static const char repomd_xml_fini[]
static void * _free(const void *p)
Wrapper to free(3), hides const compilation noise, permit NULL, return NULL.
static const char primary_xml_fini[]
static const char primary_xml_qfmt[]
char * Realpath(const char *path, char *resolved_path)
realpath(3) clone.
int rpmrepoTestSetupDirs(rpmrepo repo)
Test for repository sanity.
static const char filelists_xml_fini[]
static int rpmrepoWriteMDFile(rpmrepo repo, rpmrfile rfile, Header h)
Export a single package's metadata to repository metadata file(s).
static struct poptOption _rpmrepoOptions[]
const unsigned char * digest
int Unlink(const char *path)
unlink(2) clone.