jmq
/
Documentation
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Compare commits

base: jmq:c04e9cf3e8279b7e95589aa3c0d0136d20fedae3
Branches Tags
jmq:master
...
compare: jmq:495ce0457f1b835757b89ad5d72dbb55858c7b61
Branches Tags
jmq:master

2 Commits
c04e9cf3e8 ... 495ce0457f

Author SHA1 Message Date
John McQuah 495ce0457f prtsweep.pl: fall back to md5sums if signature file is not found 2023-08-07 08:50:12 -04:00
John McQuah 9ca23cc68d prtwash.pl: use a hash map to test files against the keeplist 2023-08-07 08:46:44 -04:00
3 changed files with 149 additions and 144 deletions
Show Stats Expand all files Collapse all files

4
man1/prtwash.1
View File

@ -115,8 +115,8 @@ algorithm makes it easy to wash all your locally-curated repositories, in contra
to the algorithm in \fBprtsweep\fP(1) which looks at the sup files used by \fBports\fP(8). to the algorithm in \fBprtsweep\fP(1) which looks at the sup files used by \fBports\fP(8).
.PP .PP
Another contrast between the two tools is that \fBprtwash\fP sources each Pkgfile to Another contrast between the two tools is that \fBprtwash\fP sources each Pkgfile to
generate its keep list, whereas \fBprtsweep\fP reads the signatures file. Spawning an generate its keep list, whereas \fBprtsweep\fP reads the SHA256 or MD5 sums from a manifest.
external bash shell for each port should in theory make \fBprtwash\fP slower Spawning an external bash shell for each port should in theory make \fBprtwash\fP slower
than \fBprtsweep\fP (which does everything in native perl), but on modern hardware the than \fBprtsweep\fP (which does everything in native perl), but on modern hardware the
difference is basically undetectable. difference is basically undetectable.
.PP .PP

282
scripts/prtsweep.pl
View File

@ -24,22 +24,22 @@ parse_args();
print_usage() if ((2*$argports-1)*(1-2*$options{auto}) < 0); print_usage() if ((2*$argports-1)*(1-2*$options{auto}) < 0);
if ($options{auto} == 1) { if ($options{auto} == 1) {
my @basedirs = getportdirs(); my @basedirs = getportdirs();
foreach my $collection (@basedirs) { foreach my $collection (@basedirs) {
print "====> Sweeping port collection $collection\n"; print "====> Sweeping port collection $collection\n";
foreach my $port (list_subdirs($collection)) { foreach my $port (list_subdirs($collection)) {
do_sweep($port); do_sweep($port);
} }
} }
} else { } else {
foreach my $port (@portdirs) { foreach my $port (@portdirs) {
do_sweep($port); do_sweep($port);
} }
} }
######################### subroutines ################################# ######################### subroutines #################################
sub print_usage { sub print_usage {
print <<EOT; print <<EOT;
Usage: prtsweep [OPTION]... [PORTDIRS]... Usage: prtsweep [OPTION]... [PORTDIRS]...
-a automatic mode, only valid when [PORTDIRS] are omitted -a automatic mode, only valid when [PORTDIRS] are omitted
@ -50,157 +50,163 @@ Usage: prtsweep [OPTION]... [PORTDIRS]...
Report bugs on libera.chat #crux-devel Report bugs on libera.chat #crux-devel
EOT EOT
exit(0); exit(0);
} }
sub parse_args { sub parse_args {
foreach my $arg (@ARGV) { foreach my $arg (@ARGV) {
if ($arg eq "-a") { if ($arg eq "-a") {
$options{auto} = 1; $options{auto} = 1;
} elsif ($arg eq "-d") { } elsif ($arg eq "-d") {
$options{rmdir} = 1; $options{rmdir} = 1;
} elsif ($arg eq "-n") { } elsif ($arg eq "-n") {
$options{dryrun} = 1; $options{dryrun} = 1;
} elsif ($arg eq "-p") { } elsif ($arg eq "-p") {
$options{pkgtoo} = 1; $options{pkgtoo} = 1;
} elsif ($arg eq "-q") { } elsif ($arg eq "-q") {
$options{quiet} = 1; $options{quiet} = 1;
} elsif ($arg eq "--version") { } elsif ($arg eq "--version") {
print $version."\n"; print $version."\n";
exit 0; exit 0;
} elsif ((-d "$arg") || # false for symlink to a portdir, } elsif ((-d "$arg") || # false for symlink to a portdir,
(-f "$arg/.signature")) { # so a second test is needed (-f "$arg/.signature") ||
push (@portdirs, $arg); (-f "$arg/.md5sum")) { # so further tests are needed
} elsif (-f "$arg") { push (@portdirs, $arg);
print "WARN: $arg is not a port directory or recognized option, ignoring.\n"; } elsif (-f "$arg") {
} else { print "WARN: $arg is not a port directory or recognized option, ignoring.\n";
print_usage(); } else {
} print_usage();
} }
$argports = @portdirs; }
$argports = @portdirs;
} }
sub list_subdirs { sub list_subdirs {
my $path = shift; my $path = shift;
my @list; my @list;
while ($path =~ s/\/\//\//g) {} while ($path =~ s/\/\//\//g) {}
$path =~ s/\/$//; $path =~ s/\/$//;
opendir(DIR, $path) or return; opendir(DIR, $path) or return;
foreach my $entry(sort(readdir(DIR))) { foreach my $entry(sort(readdir(DIR))) {
next if ( substr($entry,0,1) eq '.' ); next if ( substr($entry,0,1) eq '.' );
push (@list, "$path/$entry") if ((-d "$path/$entry") push (@list, "$path/$entry") if ((-d "$path/$entry")
or (-f "$path/$entry/.signature")); or (-f "$path/$entry/.signature"));
} }
closedir(DIR); closedir(DIR);
return @list; return @list;
} }
sub parse_signature { sub parse_manifest {
my @signed = ("Pkgfile",".footprint",".signature","README","README.md", my $sigfile = shift;
"pre-install","post-install",".32bit",".nostrip"); my $sigtype = (split /\//, $sigfile)[-1];
my $sigfile = shift; my @keeplist = ("Pkgfile",".footprint","README","README.md",
open (FILE, $sigfile) or return @signed; "pre-install","post-install",".32bit",".nostrip");
while (<FILE>) { push (@keeplist,$sigtype);
if (/^SHA256 \(.+\) =.*$/) { open (FILE, $sigfile) or return @keeplist;
$_ =~ s/^SHA256 \((.+)\) =.*$/$1/; while (<FILE>) {
push (@signed, $_) if (($sigtype eq ".signature") and (/^SHA256 \((.+)\) =.*$/)) {
} push (@keeplist, $1);
} } elsif ($sigtype eq ".md5sum") {
close (FILE); my ($m, $f) = split /\s+/, $_;
return @signed ; push (@keeplist, $f);
}
}
close (FILE);
return @keeplist ;
} }
sub sweep { sub sweep {
my $port = shift; my $port = shift; my $sigtype = shift;
while ($port =~ s/\/\//\//g) {} while ($port =~ s/\/\//\//g) {}
$port =~ s/\/$//; $port =~ s/\/$//;
my @path = split /\//, $port; my @path = split /\//, $port;
print "=======> $port\n" unless $options{quiet}==1; print "=======> $port\n" unless $options{quiet}==1;
my @wanted = parse_signature ("$port/.signature"); my %wanted = map { $_ => 1 } parse_manifest ("$port/.$sigtype");
my $builtpkg=$path[-1].'#.*pkg\.tar\.(bz2|gz|lz|xz)$'; my $builtpkg=$path[-1].'#.*pkg\.tar\.(bz2|gz|lz|xz)$';
$builtpkg =~ s/\+/\\\+/; # plus sign in filenames interferes with regex search $builtpkg =~ s/\+/\\\+/; # plus sign in filenames interferes with regex search
opendir (DIR, $port) or return; opendir (DIR, $port) or return;
foreach my $f (sort(readdir(DIR))) { foreach my $f (sort(readdir(DIR))) {
next if ( $f eq '.' or $f eq '..' ); next if ( $f eq '.' or $f eq '..' );
$f =~ s/\+/\\\+/; if (($wanted{$f}) or
if ((grep /$f/, @wanted) >= 1 or ($f =~ /$builtpkg/)*($options{pkgtoo}==0)) {
($f =~ /$builtpkg/)*($options{pkgtoo}==0)) { print "... keeping file $port/$f.\n" unless $options{quiet} == 1;
print "... keeping file $port/$f.\n" unless $options{quiet} == 1; } else {
} else { remove ("$port/$f");
remove ("$port/$f"); }
} }
} closedir (DIR);
closedir (DIR);
} }
sub remove { sub remove {
my $path=shift; my $path=shift;
my $append = ($options{dryrun}==1) ? "(dry run)\n" : "\n"; my $append = ($options{dryrun}==1) ? "(dry run)\n" : "\n";
if (-d $path) { if (-d $path) {
print "+ removing directory $path $append"; print "+ removing directory $path $append";
rmtree ($path,0,1) if ($options{dryrun}==0); rmtree ($path,0,1) if ($options{dryrun}==0);
} else { } else {
print "+ removing file $path $append"; print "+ removing file $path $append";
if ($options{dryrun}==0) { unlink "$path" or return; } if ($options{dryrun}==0) { unlink "$path" or return; }
} }
} }
sub do_sweep { sub do_sweep {
# argument either a real directory (not symlink) or has a signature; # argument either a real directory (not symlink) or has a manifest;
# this subroutine determines which condition was satisfied. # this subroutine determines which condition was satisfied.
my $port = shift; my $nf = 0; my $port = shift; my $nf = 0;
if (! -f "$port/.signature") { if ((! -f "$port/.signature") and (! -f "$port/.md5sum")) {
opendir (PORTDIR,$port) or return; opendir (PORTDIR,$port) or return;
foreach my $f (readdir PORTDIR) { foreach my $f (readdir PORTDIR) {
next if ($f eq '.' or $f eq '..'); next if ($f eq '.' or $f eq '..');
$nf += 1; $nf += 1;
} }
closedir (PORTDIR); closedir (PORTDIR);
print "WARN: $port/.signature not found, invalid port directory."; print "WARN: no signature or md5sum found in directory $port, skipping.\n";
rm_emptydir($port,$nf); rm_emptydir($port,$nf);
} else { } elsif (-f "$port/.signature") {
sweep($port); sweep($port,"signature");
} } else {
sweep($port,"md5sum");
}
} }
sub rm_emptydir { sub rm_emptydir {
my $port = shift; my $nf = shift; my $port = shift; my $nf = shift;
my $msg = ($options{rmdir}==1) ? "\n": my $msg = ($options{rmdir}==1) ? "\n":
"\n Use -d to remove empty directories.\n"; "\n Use -d to remove empty directories.\n";
my $modal = ($options{dryrun}==0) ? "" : "would be"; my $modal = ($options{dryrun}==0) ? "" : "would be";
my $post = ($nf == 0) ? " Empty directory $port $modal deleted.\n" : my $post = ($nf == 0) ? " Empty directory $port $modal deleted.\n" :
" Cannot remove $port: directory not empty\n"; " Cannot remove $port: directory not empty\n";
$msg = ($options{rmdir}==1) ? "$msg $post" : $msg ; $msg = ($options{rmdir}==1) ? "$msg $post" : $msg ;
print $msg; print $msg;
rmdir ($port) if (($nf == 0) and ($options{dryrun} == 0)); rmdir ($port) if (($nf == 0) and ($options{dryrun} == 0));
} }
sub getportdirs { sub getportdirs {
my $collection; my $collection;
my @basedirs; my @basedirs;
my $portetc = "/etc/ports/"; my $portetc = "/etc/ports/";
opendir (PORTS_DEFS,$portetc) or die "cannot open $portetc for reading"; opendir (PORTS_DEFS,$portetc) or die "cannot open $portetc for reading";
foreach (readdir PORTS_DEFS) { foreach (readdir PORTS_DEFS) {
next if ($_ eq '.' or $_ eq '..'); next if ($_ eq '.' or $_ eq '..');
if (/.*(rsync|httpup)$/) { if (/.*(rsync|httpup)$/) {
open SYNC, $portetc.$_ or die "cannot open $portetc.$_"; open SYNC, $portetc.$_ or die "cannot open $portetc.$_";
while (<SYNC>) { while (<SYNC>) {
$collection=$2 if /^(destination|ROOT_DIR)=(.+)$/; $collection=$2 if /^(destination|ROOT_DIR)=(.+)$/;
} }
close SYNC; close SYNC;
push (@basedirs , $collection); push (@basedirs , $collection);
} elsif (/.*git$/) { } elsif (/.*git$/) {
open SYNC, $portetc.$_ or die "cannot open $portetc.$_"; open SYNC, $portetc.$_ or die "cannot open $portetc.$_";
while (<SYNC>) { while (<SYNC>) {
$collection="/usr/ports/$1" if /^NAME=(.+)$/; $collection="/usr/ports/$1" if /^NAME=(.+)$/;
} }
close SYNC; close SYNC;
push (@basedirs , $collection); push (@basedirs , $collection);
} else {} } else {}
} }
closedir PORTS_DEFS; closedir PORTS_DEFS;
return @basedirs ; return @basedirs ;
} }

7
scripts/prtwash.pl
View File

@ -180,17 +180,16 @@ sub do_wash {
print "WARN: no Pkgfile found in $port. Skipping.\n"; print "WARN: no Pkgfile found in $port. Skipping.\n";
return; return;
} else { } else {
my @keepers = keeplist($port); my $iswanted; my @keepers = keeplist($port);
my $allbuilds = pop(@keepers); my $allbuilds = pop(@keepers);
my $currbuild = pop(@keepers); my $currbuild = pop(@keepers);
my %iswanted = map { $_ => 1 } @keepers;
opendir (DIR,$port) or return; opendir (DIR,$port) or return;
print "=====> washing $port\n" unless $options{quiet} == 1; print "=====> washing $port\n" unless $options{quiet} == 1;
foreach my $f (sort(readdir(DIR))) { foreach my $f (sort(readdir(DIR))) {
next if ($f eq '.' or $f eq '..'); next if ($f eq '.' or $f eq '..');
$f =~ s/\+/\\\+/g; # plus sign in filenames interferes with regex search if ($iswanted{$f} or ($options{pkgtoo}==0)*($f =~ /$currbuild/)
$iswanted = ( grep (/$f/, @keepers ) >= 1 );
if ($iswanted or ($options{pkgtoo}==0)*($f =~ /$currbuild/)
or ($options{oldver}==0)*($f =~ /$allbuilds/)*($f !~ /$currbuild/)) { or ($options{oldver}==0)*($f =~ /$allbuilds/)*($f !~ /$currbuild/)) {
print "... keeping file $port/$f.\n" unless $options{quiet} == 1; print "... keeping file $port/$f.\n" unless $options{quiet} == 1;
} else { } else {