pkg-repgen: merge globbed packages into the old index more intelligently

pkg-get: ensure all variables in status messages are initialized
This commit is contained in:
John McQuah 2023-09-18 20:46:14 +00:00
parent 1697d4af31
commit 44f3da7c5a
2 changed files with 203 additions and 118 deletions

View File

@ -32,7 +32,7 @@ my %locked; my %installed;
# CL Options
my $download_only; my $pre_install; my $post_install;
my $install_scripts; my $filter = ""; my $all; my $unused;
my $aargs=""; my $ignore_md5sum; my $force; my $root;
my $aargs=""; my $ignore_md5sum; my $force; my $root="";
GetOptions("do"=>\$download_only,
"pre-install"=>\$pre_install, "post-install"=>\$post_install,
"install-scripts"=>\$install_scripts, "all"=>\$all,
@ -428,7 +428,7 @@ sub installpkg {
if ($pkg{'readme'} eq "yes") {$readmetasks{$pkg{'name'}} = 1};
$pptasks{$pkg{'name'}} = "";
if ($force){$aa = $aa."-f ";}
if ($root) {$aa = $aa."-r ".$root." ";}
if ($root ne "") {$aa = $aa."-r ".$root." ";}
if ($install_scripts or $pre_install) {doscript("pre",%pkg);}
my $fullpath = $pkg{'path'}."/".$pkg{'name'}."#".$pkg{'version'}."-".$pkg{'release'}.".pkg.tar.$compress";
print "pkg-get: /usr/bin/pkgadd $upgrade $aa$fullpath\n";
@ -441,7 +441,7 @@ sub installpkg {
sub doscript {
my ($when, %pkg) = @_;
($pkg{$when . "_install"} eq "yes") or return;
my $cmd = ($root) ? "chroot $root " : "";
my $cmd = ($root ne "") ? "chroot $root " : "";
$cmd .= "/bin/bash $pkg{'path'}/PKGINST $pkg{'name'} $when";
if ((-e "$root$pkg{'path'}/PKGINST") and (system($cmd) == 0)) {
$pptasks{$pkg{'name'}} .= " [$when: ok]";
@ -575,7 +575,7 @@ sub remove {
$curraction = "removed";
shift(@ARGV);
my $cmd = "/usr/bin/pkgrm";
$cmd .= " -r $root" if ($root);
$cmd .= " -r $root" if ($root ne "");
foreach my $pkg(@ARGV) {
$pptasks{$pkg} = "";
if (system("$cmd $pkg")==0) {
@ -813,7 +813,7 @@ sub upinst {
my ($cmd, @args) = @_; my $aa;
($curraction, $aa) = ($cmd =~ /^up/) ? ("updated","-u") : ("installed","");
if ($root) {
if ($root ne "") {
foreach my $repo(@repos) {
my ($dir, $url) = split(/\|/, $repo);
( -e "$root$dir" ) or make_path("$root$dir");

View File

@ -29,7 +29,7 @@ $compress =~ s/["' ]//g;
$prtget .= " --no-std-config --config-set=\"prtdir $prtdir\"" if ($prtdir);
my @packages; my @dirlist; my %isDup;
my @packages; my %isDup;
sub pkg_mtime {
my $aName = $a; my $bName = $b;
@ -46,33 +46,38 @@ sub pkg_mtime {
else { return 1; }
}
if ($#ARGV > 1) { # single packages
my ($pkgdir, @args) = @ARGV;
chdir($pkgdir) or die "cannot chdir to $pkgdir!";
foreach my $name (sort @args) {
my @hits = glob("$name#*.pkg.tar.$compress");
push(@packages,@hits);
my $pkgdir = shift @ARGV; my $quickMode=0;
if (($pkgdir) and (! -d "$pkgdir")) {
print "usage: pkg-repgen [options] [directory [pkgname1..pkgnameN]]\n";
exit 1;
}
my @dirlist = ("$pkgdir") ? glob("$pkgdir/*.pkg.tar.$compress") : glob("*.pkg.tar.$compress");
@dirlist = sort pkg_mtime @dirlist;
%isDup = map { $_ => 0 } @dirlist;
if (@ARGV) { # individual packages
$quickMode=1;
my @updates = sort @ARGV;
while (my $name = shift @updates) {
push @packages, grep { m/$name#.*\.pkg/ } @dirlist;
}
} else { # one or zero arguments passed
($#ARGV == 0) or chdir($ARGV[0]) or die "cannot chdir to $ARGV[0]!";
@dirlist = glob("*.pkg.tar.$compress");
@packages = sort pkg_mtime @dirlist;
%isDup = map { $_ => 0 } @packages;
} else { # the entire directory
@packages = @dirlist;
}
# hashes to determine the package name ...
our %pname = map { $_ => $_ } @packages;
foreach my $p (@packages) { $pname{$p} =~ s/\#.*//; }
our %pname = map { $_ => (split /\//, $_)[-1] } @dirlist;
foreach my $p (@dirlist) { $pname{$p} =~ s/\#.*//; }
# ... or to look up the successor when merging old metadata files
my %followR; my %followH; my %followD; my @queue = @packages;
my %followR; my %followH; my %followD; my @queue = @dirlist;
while (my $q = shift @queue) {
($#queue < 0) or ($pname{$q} ne $pname{$queue[0]}) or $isDup{$q} = 1;
}
# Populate some other hashes using a single run of prt-get
our %path; our %depends; our %descrip; our %flags;
my @validkeys = @dirlist;
my @validkeys = map { (split /\//, $_)[-1] } @dirlist;
map { s/\#.*// } @validkeys;
my %printme = map { $_ => 1 } @validkeys;
open (my $ppf, "$prtget printf '%n^%p^%e^%d^%E:%O:%R\n' |");
@ -92,7 +97,7 @@ close ($ppf);
my %parity = ( 0 => "even", 1 => "odd" );
# Generate the metadata files
($#ARGV >= 1) ? pkg_single() : pkg_dir();
($quickMode) ? pkg_single() : pkg_dir();
# Generate README and PKGINST
pkgreadscripts();
@ -101,112 +106,185 @@ pkgreadscripts();
sub pkg_single {
my ($oR, $oD, $oH, $nR, $nD, $nH, $oline, $oname);
my $count = 0; # needed for the html index
my @dep_packages = @packages;
my @idx_packages = @packages;
my %firstrun = map { $_ => 0 } ("PKGREPO", "PKGDEPS", "index.html");
open ($oR, "PKGREPO") or $firstrun{"PKGREPO"} = 1;
open ($oD, "PKGDEPS") or $firstrun{"PKGDEPS"} = 1;
open ($oH, "index.html") or $firstrun{"index.html"} = 1;
open ($nR, ">PKGREPO.new");
open ($nD, ">PKGDEPS.new");
open ($oR, "$pkgdir/PKGREPO") or $firstrun{"PKGREPO"} = 1;
open ($oD, "$pkgdir/PKGDEPS") or $firstrun{"PKGDEPS"} = 1;
open ($oH, "$pkgdir/index.html") or $firstrun{"index.html"} = 1;
printheader(1);
open ($nH, ">>index.html.new");
foreach my $mf ("repository", "dependency map", "html index") {
print "+ Updating specified entries in $mf\n";
}
PACKAGE: while (my $p =shift @packages) {
my ($pver, $url, $du, $md5, $desc, $ppr, $pdeps, $date) = metadata($p);
($firstrun{"PKGREPO"}==0) or printf $nR "%-s:%-s:%-s:%-s:%-s\n",
$p, $du, $md5, $desc, $ppr;
($firstrun{"PKGDEPS"}==0) or ($pdeps eq "") or ($isDup{$p})
or printf $nD "%-30s : %-s\n", $pname{$p}, $pdeps;
if ($firstrun{"index.html"} == 1) {
$count++;
htmlrow($nH,$count,$pname{$p},$url,$pver,$desc,$date);
open ($nR, ">$pkgdir/PKGREPO.new");
print "+ Updating specified entries in repository\n";
RPKG: while (my $p =shift @packages) {
my ($basename, $du, $md5, $ppr) = repodata($p);
my $desc = (! $descrip{$pname{$p}}) ? "N.A." : $descrip{$pname{$p}};
if ($firstrun{"PKGREPO"}==1) {
printf $nR "%-s:%-s:%-s:%-s:%-s\n",$basename, $du, $md5, $desc, $ppr;
next RPKG;
}
($firstrun{"PKGREPO"}*$firstrun{"PKGDEPS"}*$firstrun{"index.html"}==0) or next PACKAGE;
# Pop entries from the old repository until we reach an entry
# that would come after the current package.
while ( ($firstrun{"PKGREPO"}==0) and $oline = <$oR> ) {
# Shift entries from the old repository until we find
# a successor to the current package.
while ( (! $followR{$pname{$p}}) and $oline = <$oR> ) {
chomp($oline); $oname = $oline;
$oname =~ s/\#.*//;
print $nR "$oline\n" if ($oname lt $pname{$p});
# before breaking out of the loop, either overwrite the old
# entry in the repository, or insert the requested package
# where it should appear.
printf $nR "%-s:%-s:%-s:%-s:%-s\n", $p, $du, $md5, $desc, $ppr
if ($oname ge $pname{$p});
# before breaking out of the loop, append all the packages
# from the globbed queue that are lexographically earlier
# than the current entry in the old repository.
while ($pname{$p} le $oname) {
printf $nR "%-s:%-s:%-s:%-s:%-s\n", $basename, $du, $md5, $desc, $ppr;
next RPKG if (! $isDup{$p});
$p = shift @packages;
($basename, $du, $md5, $ppr) = repodata($p);
$desc = (! $descrip{$pname{$p}}) ? "N.A." : $descrip{$pname{$p}};
# save what got popped from the repository, in case of dups
$followR{$pname{$p}} = $oline if ($oname gt $pname{$p});
# stop reading the repository, at least until the next package
last if ($oname ge $pname{$p});
# save what got shifted from the repository if we're not going to
# print it now, but don't save packages that match the same glob.
$followR{$pname{$p}} = "$oline\n" if ($pname{$p} lt $oname);
}
}
# if the current package comes after everything in the old repository,
# just append its metadata
($followR{$pname{$p}}) or printf $nR "%-s:%-s:%-s:%-s:%-s\n", $p, $du, $md5, $desc, $ppr;
($followR{$pname{$p}}) or printf $nR "%-s:%-s:%-s:%-s:%-s\n", $basename, $du, $md5, $desc, $ppr;
# Likewise for the html index
while ( ($firstrun{"index.html"}==0) and $oline=<$oH> ) {
chomp($oline);
# no need to copy the header, it should already be there
next if ($oline !~ m/^<tr class="(odd|even)"/);
$count++;
$oname = $oline;
$oname =~ s/.*a href="(.*)"/$1/;
$oname =~ s/\%23.*//;
next RPKG if (($isDup{$p}) or (! $followR{$pname{$p}}));
print $nH "$oline\n" if ($oname lt $pname{$p});
htmlrow($nH,$count,$pname{$p},$url,
$pver,$desc,$date) if ($oname ge $pname{$p});
$followH{$pname{$p}} = $oline if ($oname gt $pname{$p});
last if ($oname ge $pname{$p});
# Arriving here means the current package is not a dup, and
# definitely has a successor in the old repository. But the
# next globbed package might be a more immediate successor.
# Decide which of the two possible successors comes first.
# By defining a successor for the next package in the queue,
# we delay shifting entries off the old repo.
if ((@packages) and ($pname{$packages[0]} le $followR{$pname{$p}})) {
$followR{$pname{$packages[0]}} = $followR{$pname{$p}};
next RPKG;
} else {
print $nR $followR{$pname{$p}};
}
if (! $followH{$pname{$p}}) {
# Shift another package from the queue
}
# Likewise for the html index
printheader(1);
open ($nH, ">>$pkgdir/index.html.new");
print "+ Updating specified entries in the html index\n";
HPKG: while (my $p =shift @idx_packages) {
my ($url, $pver, $desc, $date) = htmldata($p);
if ($firstrun{"index.html"} == 1) {
$count++;
htmlrow($nH,$count,$pname{$p},$url,$pver,$desc,$date);
next HPKG;
}
# Shift entries from the old html index until we find
# a successor to the current package.
while ( (! $followH{$pname{$p}}) and $oline=<$oH> ) {
chomp($oline);
# no need to copy the header, it should already be there
next if ($oline !~ m/^<tr class="(odd|even)"/);
$oname = $oline;
$oname =~ s/.*a href="(.*)"/$1/; $oname =~ s/\%23.*//;
if ($oname lt $pname{$p}) { $count++; print $nH "$oline\n"; }
# before breaking out of the loop, append all the packages
# from the globbed queue that are lexographically earlier
# than the current entry in the old html index.
while ($pname{$p} le $oname) {
$count++;
htmlrow($nH,$count,$pname{$p},$url,$pver,$desc,$date);
next HPKG if (! $isDup{$p});
$p = shift @idx_packages;
($url, $pver, $desc, $date) = htmldata($p);
# save what got shifted from the index if we're not going to print
# it now, but ignore packages that match the same glob.
$followH{$pname{$p}} = "$oline\n" if ($pname{$p} lt $oname);
}
}
# Likewise for the dependency map, but avoid creating duplicate entries
while ( ($firstrun{"PKGDEPS"}==0) and $oline = <$oD> ) {
# if the current package comes after everything in the old html index,
# just append its metadata
if (! $followH{$pname{$p}}) {
$count++;
htmlrow($nH,$count,$pname{$p},$url,$pver,$desc,$date);
}
next HPKG if (($isDup{$p}) or (! $followH{$pname{$p}}));
# Arriving here means the current package is not a dup, and
# definitely has a successor in the old html index. But the
# next globbed package might be a more immediate successor.
# Decide which of the two possible successors comes first. If the
# globbed package is the more immediate successor, use its name
# as a key to retain the most recent entry from the html index.
if ((@packages) and ($pname{$packages[0]} le $followH{$pname{$p}})) {
$followH{$pname{$packages[0]}} = $followH{$pname{$p}};
next HPKG;
} else {
$count++;
$followH{$pname{$p}} =~ s/class="(even|odd)"/class="$parity{($count %2)}"/;
print $nH $followH{$pname{$p}};
}
# Shift another package from the queue
}
# Likewise for the dependency map, but avoid creating duplicate entries
open ($nD, ">$pkgdir/PKGDEPS.new");
print "+ Updating specified entries in the depmap\n";
DPKG: while (my $p =shift @dep_packages) {
if ($firstrun{"PKGDEPS"}==1) {
(! $depends{$pname{$p}}) or ($isDup{$p})
or printf $nD "%-30s : %-s\n", $pname{$p}, $depends{$pname{$p}};
next DPKG;
}
# Shift entries from the old depmap until we find a successor
# to the current package
while ( (! $followD{$pname{$p}}) and $oline = <$oD> ) {
chomp($oline); $oname = $oline;
$oname =~ s/\s*\:.*//;
if ($oname lt $pname{$p}) {
print $nD "$oline\n";
} elsif ( ($pdeps ne "") and (! $isDup{$p}) ) {
printf $nD "%-30s : %-s\n", $pname{$p}, $pdeps;
}
if ($oname gt $pname{$p}) {
$followD{$pname{$p}} = $oline;
print $nD "$oline\n";
}
last if ($oname ge $pname{$p});
print $nD "$oline\n" if ($oname lt $pname{$p});
while ($pname{$p} le $oname) {
if (! $isDup{$p}) {
printf $nD "%-30s : %-s\n", $pname{$p}, $depends{$pname{$p}};
next DPKG;
} else {
$p = shift @dep_packages;
}
# save what got shifted from the depmap if we're not going to print
# it now, but ignore packages that match the same glob.
$followD{$pname{$p}} = $oline if ($pname{$p} lt $oname);
}
}
# if the current package comes after everything in the old depmap,
# just append its metadata
($followD{$pname{$p}}) or ($isDup{$p}) or ($pdeps eq "")
or printf $nD "%-30s : %-s\n", $pname{$p}, $pdeps;
# if the current package comes after everything in the old depmap
# and is not a dup, just append its metadata
($followD{$pname{$p}}) or ($isDup{$p}) or (! $depends{$pname{$p}})
or printf $nD "%-30s : %-s\n", $pname{$p}, $depends{$pname{$p}};
# after reaching the last in a sequence of dups, copy the
# successor line from the old {html index, repository}
if ( (! $isDup{$p}) and ($followH{$pname{$p}}) ) {
$count++;
$followH{$pname{$p}} =~ s/class="(even|odd)"/class="$parity{($count %2)}"/;
print $nH $followH{$pname{$p}};
next DPKG if (($isDup{$p}) or (! $followD{$pname{$p}}));
# Arriving here means the current package is not a dup, and
# definitely has a successor entry in the old depmap.
# But the next globbed package might be a more immediate successor.
# Decide which of the two possible successors comes first. If it's the
# globbed package that comes next, save the old depmap entry.
if ((@packages) and ($pname{$packages[0]} le $followD{$pname{$p}})) {
$followD{$pname{$packages[0]}} = $followD{$pname{$p}};
next DPKG;
} else {
printf $nD $followD{$pname{$p}};
}
($isDup{$p}) or (! $followR{$pname{$p}}) or print $nR $followR{$pname{$p}};
# Restart the loop with the next package in the queue
# Shift another package from the queue
}
# Done with all the packages that match command-line arguments.
@ -226,24 +304,25 @@ sub pkg_single {
($firstrun{"PKGDEPS"}==1) or close($oD);
($firstrun{"index.html"}==1) or close($oH);
foreach my $db (keys %firstrun) { rename("$db.new", "$db"); }
foreach my $db (keys %firstrun) { rename("$pkgdir/$db.new", "$pkgdir/$db"); }
printfooter($count) if ($firstrun{"index.html"} == 1);
}
######################## full repository ########################
sub pkg_dir {
print "+ Generating dependencies\n";
open (my $iD, ">PKGDEPS");
open (my $iD, ">$pkgdir/PKGDEPS");
print "+ Generating repository\n";
open (my $iR, ">PKGREPO");
open (my $iR, ">$pkgdir/PKGREPO");
printheader(0);
my $count = 0;
open (my $ih, '>>index.html');
open (my $ih, ">>$pkgdir/index.html");
foreach my $p (@packages) {
my ($pver, $url, $du, $md5, $desc, $ppr, $pdeps, $date) = metadata($p);
($pdeps eq "") or ($isDup{$p})
or printf $iD "%-30s : %-s\n", $pname{$p}, $pdeps;
printf $iR "%-s:%-s:%-s:%-s:%-s\n", $p,$du,$md5,$desc,$ppr;
my ($basename, $du, $md5, $ppr) = repodata($p);
my ($url, $pver, $desc, $date) = htmldata($p);
(! $depends{$pname{$p}}) or ($isDup{$p})
or printf $iD "%-30s : %-s\n", $pname{$p}, $depends{$pname{$p}};
printf $iR "%-s:%-s:%-s:%-s:%-s\n", $basename,$du,$md5,$desc,$ppr;
$count++;
htmlrow($ih,$count,$pname{$p},$url,$pver,$desc,$date);
}
@ -256,11 +335,11 @@ sub pkg_dir {
# consolidate all the README and install scripts for the available packages
sub pkgreadscripts {
print "+ Generating README\n";
open (my $fR, '>PKGREAD');
open (my $fR, ">$pkgdir/PKGREAD");
print $fR "# README files for repository. Do NOT remove this line.\n";
print "+ Generating scripts\n";
open (my $fS, '>PKGINST');
open (my $fS, ">$pkgdir/PKGINST");
print $fS '#!/usr/bin/env bash
#
# PKGINST: pre- and post-install scripts for CRUX packages
@ -271,7 +350,8 @@ run_script() {
my %seen;
foreach my $name (@dirlist) {
$name =~ s/\#.*//; next if ($seen{$name});
$name =~ s/\#.*//; $name = (split /\//, $name)[-1];
next if ($seen{$name});
$seen{$name} = 1;
next if (! $path{$name});
if (-f "$path{$name}/README"){
@ -304,7 +384,7 @@ run_script() {
sub printheader {
my $isTemp = shift; my $ih;
($isTemp == 0) ? open ($ih, '>index.html') : open ($ih, '>index.html.new');
($isTemp == 0) ? open ($ih, ">$pkgdir/index.html") : open ($ih, ">$pkgdir/index.html.new");
print $ih <<EOH;
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@ -376,7 +456,7 @@ sub htmlrow {
sub printfooter {
my $count = shift;
open (my $ih, '>>index.html');
open (my $ih, ">>$pkgdir/index.html");
print $ih " </table>\n";
print $ih " <p><b>$count packages</b></p>\n";
if ($footer) {
@ -392,18 +472,23 @@ EOH
close($ih);
}
sub metadata {
sub htmldata {
my $p = shift;
my ($pver, $url) = ($p, $p);
$pver =~ s/.*\#//; $pver =~ s/\.pkg\.tar.*//;
$url =~ s/\#/\%23/;
$url = (split /\//, $p)[-1]; $url =~ s/\#/\%23/;
my $date = isotime( (stat($p))[9], 1);
my $desc = (! $descrip{$pname{$p}}) ? "N.A." : $descrip{$pname{$p}};
return $url, $pver, $desc, $date;
}
sub repodata {
my $p = shift;
my $basename = (split /\//, $p)[-1];
my $du = (-s $p);
my $md5 = digest_file_hex($p,"MD5");
my $desc = (! $descrip{$pname{$p}}) ? "N.A." : $descrip{$pname{$p}};
my $ppr = (! $flags{$pname{$p}}) ? "no:no:no" : $flags{$pname{$p}};
my $pdeps = (! $depends{$pname{$p}}) ? "" : $depends{$pname{$p}};
my $date = isotime( (stat($p))[9], 1);
return $pver, $url, $du, $md5, $desc, $ppr, $pdeps, $date;
return $basename, $du, $md5, $ppr;
}
sub isotime {