From 44f3da7c5a94e820e08ebf04b670c998d065adf3 Mon Sep 17 00:00:00 2001 From: John McQuah Date: Mon, 18 Sep 2023 20:46:14 +0000 Subject: [PATCH] pkg-repgen: merge globbed packages into the old index more intelligently pkg-get: ensure all variables in status messages are initialized --- scripts/pkg-get.pl | 10 +- scripts/pkg-repgen.pl | 311 +++++++++++++++++++++++++++--------------- 2 files changed, 203 insertions(+), 118 deletions(-) diff --git a/scripts/pkg-get.pl b/scripts/pkg-get.pl index 2b19e10..941ab7e 100755 --- a/scripts/pkg-get.pl +++ b/scripts/pkg-get.pl @@ -32,7 +32,7 @@ my %locked; my %installed; # CL Options my $download_only; my $pre_install; my $post_install; my $install_scripts; my $filter = ""; my $all; my $unused; -my $aargs=""; my $ignore_md5sum; my $force; my $root; +my $aargs=""; my $ignore_md5sum; my $force; my $root=""; GetOptions("do"=>\$download_only, "pre-install"=>\$pre_install, "post-install"=>\$post_install, "install-scripts"=>\$install_scripts, "all"=>\$all, @@ -428,7 +428,7 @@ sub installpkg { if ($pkg{'readme'} eq "yes") {$readmetasks{$pkg{'name'}} = 1}; $pptasks{$pkg{'name'}} = ""; if ($force){$aa = $aa."-f ";} - if ($root) {$aa = $aa."-r ".$root." ";} + if ($root ne "") {$aa = $aa."-r ".$root." ";} if ($install_scripts or $pre_install) {doscript("pre",%pkg);} my $fullpath = $pkg{'path'}."/".$pkg{'name'}."#".$pkg{'version'}."-".$pkg{'release'}.".pkg.tar.$compress"; print "pkg-get: /usr/bin/pkgadd $upgrade $aa$fullpath\n"; @@ -441,7 +441,7 @@ sub installpkg { sub doscript { my ($when, %pkg) = @_; ($pkg{$when . "_install"} eq "yes") or return; - my $cmd = ($root) ? "chroot $root " : ""; + my $cmd = ($root ne "") ? "chroot $root " : ""; $cmd .= "/bin/bash $pkg{'path'}/PKGINST $pkg{'name'} $when"; if ((-e "$root$pkg{'path'}/PKGINST") and (system($cmd) == 0)) { $pptasks{$pkg{'name'}} .= " [$when: ok]"; @@ -575,7 +575,7 @@ sub remove { $curraction = "removed"; shift(@ARGV); my $cmd = "/usr/bin/pkgrm"; - $cmd .= " -r $root" if ($root); + $cmd .= " -r $root" if ($root ne ""); foreach my $pkg(@ARGV) { $pptasks{$pkg} = ""; if (system("$cmd $pkg")==0) { @@ -813,7 +813,7 @@ sub upinst { my ($cmd, @args) = @_; my $aa; ($curraction, $aa) = ($cmd =~ /^up/) ? ("updated","-u") : ("installed",""); - if ($root) { + if ($root ne "") { foreach my $repo(@repos) { my ($dir, $url) = split(/\|/, $repo); ( -e "$root$dir" ) or make_path("$root$dir"); diff --git a/scripts/pkg-repgen.pl b/scripts/pkg-repgen.pl index 9ccb959..7bbad0e 100755 --- a/scripts/pkg-repgen.pl +++ b/scripts/pkg-repgen.pl @@ -29,7 +29,7 @@ $compress =~ s/["' ]//g; $prtget .= " --no-std-config --config-set=\"prtdir $prtdir\"" if ($prtdir); -my @packages; my @dirlist; my %isDup; +my @packages; my %isDup; sub pkg_mtime { my $aName = $a; my $bName = $b; @@ -46,33 +46,38 @@ sub pkg_mtime { else { return 1; } } -if ($#ARGV > 1) { # single packages - my ($pkgdir, @args) = @ARGV; - chdir($pkgdir) or die "cannot chdir to $pkgdir!"; - foreach my $name (sort @args) { - my @hits = glob("$name#*.pkg.tar.$compress"); - push(@packages,@hits); +my $pkgdir = shift @ARGV; my $quickMode=0; +if (($pkgdir) and (! -d "$pkgdir")) { + print "usage: pkg-repgen [options] [directory [pkgname1..pkgnameN]]\n"; + exit 1; +} +my @dirlist = ("$pkgdir") ? glob("$pkgdir/*.pkg.tar.$compress") : glob("*.pkg.tar.$compress"); +@dirlist = sort pkg_mtime @dirlist; +%isDup = map { $_ => 0 } @dirlist; + +if (@ARGV) { # individual packages + $quickMode=1; + my @updates = sort @ARGV; + while (my $name = shift @updates) { + push @packages, grep { m/$name#.*\.pkg/ } @dirlist; } -} else { # one or zero arguments passed - ($#ARGV == 0) or chdir($ARGV[0]) or die "cannot chdir to $ARGV[0]!"; - @dirlist = glob("*.pkg.tar.$compress"); - @packages = sort pkg_mtime @dirlist; - %isDup = map { $_ => 0 } @packages; +} else { # the entire directory + @packages = @dirlist; } # hashes to determine the package name ... -our %pname = map { $_ => $_ } @packages; -foreach my $p (@packages) { $pname{$p} =~ s/\#.*//; } +our %pname = map { $_ => (split /\//, $_)[-1] } @dirlist; +foreach my $p (@dirlist) { $pname{$p} =~ s/\#.*//; } # ... or to look up the successor when merging old metadata files -my %followR; my %followH; my %followD; my @queue = @packages; +my %followR; my %followH; my %followD; my @queue = @dirlist; while (my $q = shift @queue) { ($#queue < 0) or ($pname{$q} ne $pname{$queue[0]}) or $isDup{$q} = 1; } # Populate some other hashes using a single run of prt-get our %path; our %depends; our %descrip; our %flags; -my @validkeys = @dirlist; +my @validkeys = map { (split /\//, $_)[-1] } @dirlist; map { s/\#.*// } @validkeys; my %printme = map { $_ => 1 } @validkeys; open (my $ppf, "$prtget printf '%n^%p^%e^%d^%E:%O:%R\n' |"); @@ -92,7 +97,7 @@ close ($ppf); my %parity = ( 0 => "even", 1 => "odd" ); # Generate the metadata files -($#ARGV >= 1) ? pkg_single() : pkg_dir(); +($quickMode) ? pkg_single() : pkg_dir(); # Generate README and PKGINST pkgreadscripts(); @@ -101,112 +106,185 @@ pkgreadscripts(); sub pkg_single { my ($oR, $oD, $oH, $nR, $nD, $nH, $oline, $oname); my $count = 0; # needed for the html index + my @dep_packages = @packages; + my @idx_packages = @packages; my %firstrun = map { $_ => 0 } ("PKGREPO", "PKGDEPS", "index.html"); - open ($oR, "PKGREPO") or $firstrun{"PKGREPO"} = 1; - open ($oD, "PKGDEPS") or $firstrun{"PKGDEPS"} = 1; - open ($oH, "index.html") or $firstrun{"index.html"} = 1; - open ($nR, ">PKGREPO.new"); - open ($nD, ">PKGDEPS.new"); + open ($oR, "$pkgdir/PKGREPO") or $firstrun{"PKGREPO"} = 1; + open ($oD, "$pkgdir/PKGDEPS") or $firstrun{"PKGDEPS"} = 1; + open ($oH, "$pkgdir/index.html") or $firstrun{"index.html"} = 1; - printheader(1); - open ($nH, ">>index.html.new"); - - foreach my $mf ("repository", "dependency map", "html index") { - print "+ Updating specified entries in $mf\n"; - } - - PACKAGE: while (my $p =shift @packages) { - my ($pver, $url, $du, $md5, $desc, $ppr, $pdeps, $date) = metadata($p); - - ($firstrun{"PKGREPO"}==0) or printf $nR "%-s:%-s:%-s:%-s:%-s\n", - $p, $du, $md5, $desc, $ppr; - ($firstrun{"PKGDEPS"}==0) or ($pdeps eq "") or ($isDup{$p}) - or printf $nD "%-30s : %-s\n", $pname{$p}, $pdeps; - if ($firstrun{"index.html"} == 1) { - $count++; - htmlrow($nH,$count,$pname{$p},$url,$pver,$desc,$date); + open ($nR, ">$pkgdir/PKGREPO.new"); + print "+ Updating specified entries in repository\n"; + RPKG: while (my $p =shift @packages) { + my ($basename, $du, $md5, $ppr) = repodata($p); + my $desc = (! $descrip{$pname{$p}}) ? "N.A." : $descrip{$pname{$p}}; + if ($firstrun{"PKGREPO"}==1) { + printf $nR "%-s:%-s:%-s:%-s:%-s\n",$basename, $du, $md5, $desc, $ppr; + next RPKG; } - ($firstrun{"PKGREPO"}*$firstrun{"PKGDEPS"}*$firstrun{"index.html"}==0) or next PACKAGE; - # Pop entries from the old repository until we reach an entry - # that would come after the current package. - while ( ($firstrun{"PKGREPO"}==0) and $oline = <$oR> ) { + # Shift entries from the old repository until we find + # a successor to the current package. + while ( (! $followR{$pname{$p}}) and $oline = <$oR> ) { chomp($oline); $oname = $oline; $oname =~ s/\#.*//; print $nR "$oline\n" if ($oname lt $pname{$p}); - # before breaking out of the loop, either overwrite the old - # entry in the repository, or insert the requested package - # where it should appear. - printf $nR "%-s:%-s:%-s:%-s:%-s\n", $p, $du, $md5, $desc, $ppr - if ($oname ge $pname{$p}); + # before breaking out of the loop, append all the packages + # from the globbed queue that are lexographically earlier + # than the current entry in the old repository. + while ($pname{$p} le $oname) { + printf $nR "%-s:%-s:%-s:%-s:%-s\n", $basename, $du, $md5, $desc, $ppr; + next RPKG if (! $isDup{$p}); + $p = shift @packages; + ($basename, $du, $md5, $ppr) = repodata($p); + $desc = (! $descrip{$pname{$p}}) ? "N.A." : $descrip{$pname{$p}}; - # save what got popped from the repository, in case of dups - $followR{$pname{$p}} = $oline if ($oname gt $pname{$p}); - - # stop reading the repository, at least until the next package - last if ($oname ge $pname{$p}); + # save what got shifted from the repository if we're not going to + # print it now, but don't save packages that match the same glob. + $followR{$pname{$p}} = "$oline\n" if ($pname{$p} lt $oname); + } } # if the current package comes after everything in the old repository, # just append its metadata - ($followR{$pname{$p}}) or printf $nR "%-s:%-s:%-s:%-s:%-s\n", $p, $du, $md5, $desc, $ppr; + ($followR{$pname{$p}}) or printf $nR "%-s:%-s:%-s:%-s:%-s\n", $basename, $du, $md5, $desc, $ppr; - # Likewise for the html index - while ( ($firstrun{"index.html"}==0) and $oline=<$oH> ) { - chomp($oline); - # no need to copy the header, it should already be there - next if ($oline !~ m/^>$pkgdir/index.html.new"); + + print "+ Updating specified entries in the html index\n"; + HPKG: while (my $p =shift @idx_packages) { + my ($url, $pver, $desc, $date) = htmldata($p); + + if ($firstrun{"index.html"} == 1) { + $count++; + htmlrow($nH,$count,$pname{$p},$url,$pver,$desc,$date); + next HPKG; + } + # Shift entries from the old html index until we find + # a successor to the current package. + while ( (! $followH{$pname{$p}}) and $oline=<$oH> ) { + chomp($oline); + # no need to copy the header, it should already be there + next if ($oline !~ m/^ ) { + # if the current package comes after everything in the old html index, + # just append its metadata + if (! $followH{$pname{$p}}) { + $count++; + htmlrow($nH,$count,$pname{$p},$url,$pver,$desc,$date); + } + + next HPKG if (($isDup{$p}) or (! $followH{$pname{$p}})); + + # Arriving here means the current package is not a dup, and + # definitely has a successor in the old html index. But the + # next globbed package might be a more immediate successor. + # Decide which of the two possible successors comes first. If the + # globbed package is the more immediate successor, use its name + # as a key to retain the most recent entry from the html index. + if ((@packages) and ($pname{$packages[0]} le $followH{$pname{$p}})) { + $followH{$pname{$packages[0]}} = $followH{$pname{$p}}; + next HPKG; + } else { + $count++; + $followH{$pname{$p}} =~ s/class="(even|odd)"/class="$parity{($count %2)}"/; + print $nH $followH{$pname{$p}}; + } + + # Shift another package from the queue + } + + # Likewise for the dependency map, but avoid creating duplicate entries + open ($nD, ">$pkgdir/PKGDEPS.new"); + + print "+ Updating specified entries in the depmap\n"; + DPKG: while (my $p =shift @dep_packages) { + if ($firstrun{"PKGDEPS"}==1) { + (! $depends{$pname{$p}}) or ($isDup{$p}) + or printf $nD "%-30s : %-s\n", $pname{$p}, $depends{$pname{$p}}; + next DPKG; + } + # Shift entries from the old depmap until we find a successor + # to the current package + while ( (! $followD{$pname{$p}}) and $oline = <$oD> ) { chomp($oline); $oname = $oline; $oname =~ s/\s*\:.*//; - if ($oname lt $pname{$p}) { - print $nD "$oline\n"; - } elsif ( ($pdeps ne "") and (! $isDup{$p}) ) { - printf $nD "%-30s : %-s\n", $pname{$p}, $pdeps; - } - if ($oname gt $pname{$p}) { - $followD{$pname{$p}} = $oline; - print $nD "$oline\n"; - } - last if ($oname ge $pname{$p}); + print $nD "$oline\n" if ($oname lt $pname{$p}); + while ($pname{$p} le $oname) { + if (! $isDup{$p}) { + printf $nD "%-30s : %-s\n", $pname{$p}, $depends{$pname{$p}}; + next DPKG; + } else { + $p = shift @dep_packages; + } + # save what got shifted from the depmap if we're not going to print + # it now, but ignore packages that match the same glob. + $followD{$pname{$p}} = $oline if ($pname{$p} lt $oname); + } } - # if the current package comes after everything in the old depmap, - # just append its metadata - ($followD{$pname{$p}}) or ($isDup{$p}) or ($pdeps eq "") - or printf $nD "%-30s : %-s\n", $pname{$p}, $pdeps; + # if the current package comes after everything in the old depmap + # and is not a dup, just append its metadata + ($followD{$pname{$p}}) or ($isDup{$p}) or (! $depends{$pname{$p}}) + or printf $nD "%-30s : %-s\n", $pname{$p}, $depends{$pname{$p}}; - # after reaching the last in a sequence of dups, copy the - # successor line from the old {html index, repository} - if ( (! $isDup{$p}) and ($followH{$pname{$p}}) ) { - $count++; - $followH{$pname{$p}} =~ s/class="(even|odd)"/class="$parity{($count %2)}"/; - print $nH $followH{$pname{$p}}; + next DPKG if (($isDup{$p}) or (! $followD{$pname{$p}})); + + # Arriving here means the current package is not a dup, and + # definitely has a successor entry in the old depmap. + # But the next globbed package might be a more immediate successor. + # Decide which of the two possible successors comes first. If it's the + # globbed package that comes next, save the old depmap entry. + if ((@packages) and ($pname{$packages[0]} le $followD{$pname{$p}})) { + $followD{$pname{$packages[0]}} = $followD{$pname{$p}}; + next DPKG; + } else { + printf $nD $followD{$pname{$p}}; } - ($isDup{$p}) or (! $followR{$pname{$p}}) or print $nR $followR{$pname{$p}}; - # Restart the loop with the next package in the queue + # Shift another package from the queue } # Done with all the packages that match command-line arguments. @@ -226,24 +304,25 @@ sub pkg_single { ($firstrun{"PKGDEPS"}==1) or close($oD); ($firstrun{"index.html"}==1) or close($oH); - foreach my $db (keys %firstrun) { rename("$db.new", "$db"); } + foreach my $db (keys %firstrun) { rename("$pkgdir/$db.new", "$pkgdir/$db"); } printfooter($count) if ($firstrun{"index.html"} == 1); } ######################## full repository ######################## sub pkg_dir { print "+ Generating dependencies\n"; - open (my $iD, ">PKGDEPS"); + open (my $iD, ">$pkgdir/PKGDEPS"); print "+ Generating repository\n"; - open (my $iR, ">PKGREPO"); + open (my $iR, ">$pkgdir/PKGREPO"); printheader(0); my $count = 0; - open (my $ih, '>>index.html'); + open (my $ih, ">>$pkgdir/index.html"); foreach my $p (@packages) { - my ($pver, $url, $du, $md5, $desc, $ppr, $pdeps, $date) = metadata($p); - ($pdeps eq "") or ($isDup{$p}) - or printf $iD "%-30s : %-s\n", $pname{$p}, $pdeps; - printf $iR "%-s:%-s:%-s:%-s:%-s\n", $p,$du,$md5,$desc,$ppr; + my ($basename, $du, $md5, $ppr) = repodata($p); + my ($url, $pver, $desc, $date) = htmldata($p); + (! $depends{$pname{$p}}) or ($isDup{$p}) + or printf $iD "%-30s : %-s\n", $pname{$p}, $depends{$pname{$p}}; + printf $iR "%-s:%-s:%-s:%-s:%-s\n", $basename,$du,$md5,$desc,$ppr; $count++; htmlrow($ih,$count,$pname{$p},$url,$pver,$desc,$date); } @@ -256,11 +335,11 @@ sub pkg_dir { # consolidate all the README and install scripts for the available packages sub pkgreadscripts { print "+ Generating README\n"; - open (my $fR, '>PKGREAD'); + open (my $fR, ">$pkgdir/PKGREAD"); print $fR "# README files for repository. Do NOT remove this line.\n"; print "+ Generating scripts\n"; - open (my $fS, '>PKGINST'); + open (my $fS, ">$pkgdir/PKGINST"); print $fS '#!/usr/bin/env bash # # PKGINST: pre- and post-install scripts for CRUX packages @@ -271,7 +350,8 @@ run_script() { my %seen; foreach my $name (@dirlist) { - $name =~ s/\#.*//; next if ($seen{$name}); + $name =~ s/\#.*//; $name = (split /\//, $name)[-1]; + next if ($seen{$name}); $seen{$name} = 1; next if (! $path{$name}); if (-f "$path{$name}/README"){ @@ -304,7 +384,7 @@ run_script() { sub printheader { my $isTemp = shift; my $ih; - ($isTemp == 0) ? open ($ih, '>index.html') : open ($ih, '>index.html.new'); + ($isTemp == 0) ? open ($ih, ">$pkgdir/index.html") : open ($ih, ">$pkgdir/index.html.new"); print $ih < @@ -376,7 +456,7 @@ sub htmlrow { sub printfooter { my $count = shift; - open (my $ih, '>>index.html'); + open (my $ih, ">>$pkgdir/index.html"); print $ih " \n"; print $ih "

$count packages

\n"; if ($footer) { @@ -392,18 +472,23 @@ EOH close($ih); } -sub metadata { +sub htmldata { my $p = shift; my ($pver, $url) = ($p, $p); $pver =~ s/.*\#//; $pver =~ s/\.pkg\.tar.*//; - $url =~ s/\#/\%23/; + $url = (split /\//, $p)[-1]; $url =~ s/\#/\%23/; + my $date = isotime( (stat($p))[9], 1); + my $desc = (! $descrip{$pname{$p}}) ? "N.A." : $descrip{$pname{$p}}; + return $url, $pver, $desc, $date; +} + +sub repodata { + my $p = shift; + my $basename = (split /\//, $p)[-1]; my $du = (-s $p); my $md5 = digest_file_hex($p,"MD5"); - my $desc = (! $descrip{$pname{$p}}) ? "N.A." : $descrip{$pname{$p}}; my $ppr = (! $flags{$pname{$p}}) ? "no:no:no" : $flags{$pname{$p}}; - my $pdeps = (! $depends{$pname{$p}}) ? "" : $depends{$pname{$p}}; - my $date = isotime( (stat($p))[9], 1); - return $pver, $url, $du, $md5, $desc, $ppr, $pdeps, $date; + return $basename, $du, $md5, $ppr; } sub isotime {