From cea1d3a0f3b575a51e5f46c747840693bb3265ce Mon Sep 17 00:00:00 2001 From: espie Date: Fri, 18 May 2012 12:11:28 +0000 Subject: [PATCH] bite the bullet, reuse dpb's algorithm to make sure we get correct PkgPath. This implies restructuring the database code, and the start of a better separation between variables handling, and actual insertion in the database. bump version to 2.0: this changes the paths that actually get recorded, and pkglocatedb will need to be aware of that. --- databases/sqlports/Makefile | 6 +- databases/sqlports/files/Info.pm | 140 +++++++++++++++++++ databases/sqlports/files/Inserter.pm | 25 +--- databases/sqlports/files/PkgPath.pm | 66 +++++++++ databases/sqlports/files/Var.pm | 38 +++-- databases/sqlports/files/mksqlitedb | 201 +++++++++++---------------- 6 files changed, 319 insertions(+), 157 deletions(-) create mode 100644 databases/sqlports/files/Info.pm create mode 100644 databases/sqlports/files/PkgPath.pm diff --git a/databases/sqlports/Makefile b/databases/sqlports/Makefile index c534a75fb93..78d4f6bf5d1 100644 --- a/databases/sqlports/Makefile +++ b/databases/sqlports/Makefile @@ -1,7 +1,7 @@ -# $OpenBSD: Makefile,v 1.39 2012/04/29 16:34:53 espie Exp $ +# $OpenBSD: Makefile,v 1.40 2012/05/18 12:11:28 espie Exp $ CATEGORIES = databases -V = 1.20 +V = 2.0 DISTNAME = sqlports-$V DISTFILES = COMMENT = sqlite database of ports @@ -21,7 +21,7 @@ MULTI_PACKAGES = -main -compact DBNAME = ${WRKBUILD}/sqlports do-build: - @cd ${PORTSDIR} && perl ${FILESDIR}/mksqlitedb -v ${DBNAME} -p ${WRKBUILD}/ouch + @cd ${PORTSDIR} && PORTSDIR=${PORTSDIR} perl ${FILESDIR}/mksqlitedb -v ${DBNAME} -p ${WRKBUILD}/ouch @if test -s ${WRKBUILD}/ouch; then \ cat ${WRKBUILD}/ouch; \ exit 1; \ diff --git a/databases/sqlports/files/Info.pm b/databases/sqlports/files/Info.pm new file mode 100644 index 00000000000..9f9a7567781 --- /dev/null +++ b/databases/sqlports/files/Info.pm @@ -0,0 +1,140 @@ +# $OpenBSD: Info.pm,v 1.1 2012/05/18 12:11:28 espie Exp $ +# +# Copyright (c) 2012 Marc Espie +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# example script that shows how to store all variable values into a +# database, using SQLite for that purpose. +# +# usage: cd /usr/ports && make dump-vars |mksqlitedb + +use strict; +use warnings; +use Var; + +package Info; +our $vars = { + AUTOCONF_VERSION => 'AutoVersionVar', + AUTOMAKE_VERSION => 'AutoVersionVar', + BROKEN => 'BrokenVar', + BUILD_DEPENDS => 'BuildDependsVar', + CATEGORIES => 'CategoriesVar', + COMES_WITH => 'DefinedVar', + COMMENT => 'AnyVar', + CONFIGURE_ARGS => 'ConfigureArgsVar', + CONFIGURE_STYLE => 'ConfigureVar', + DESCR => 'FileVar', + DISTFILES => 'AnyVar', + PATCHFILES => 'AnyVar', + DISTNAME => 'AnyVar', + DIST_SUBDIR => 'DefinedVar', + EPOCH => 'AnyVar', + FETCH_MANUALLY => 'IgnoredVar', + FLAVOR => 'IgnoredVar', + FLAVORS => 'FlavorsVar', + FULLPKGNAME => 'AnyVar', + HOMEPAGE => 'AnyVar', + IGNORE => 'DefinedVar', + IS_INTERACTIVE => 'AnyVar', + LIB_DEPENDS => 'LibDependsVar', + MAINTAINER=> 'EmailVar', + MASTER_SITES => 'MasterSitesVar', + MASTER_SITES0 => 'MasterSitesVar', + MASTER_SITES1 => 'MasterSitesVar', + MASTER_SITES2 => 'MasterSitesVar', + MASTER_SITES3 => 'MasterSitesVar', + MASTER_SITES4=> 'MasterSitesVar', + MASTER_SITES5 => 'MasterSitesVar', + MASTER_SITES6 => 'MasterSitesVar', + MASTER_SITES7 => 'MasterSitesVar', + MASTER_SITES8 => 'MasterSitesVar', + MASTER_SITES9=> 'MasterSitesVar', + MISSING_FILES => 'IgnoredVar', + MODULES => 'ModulesVar', + MULTI_PACKAGES => 'MultiVar', + NO_BUILD => 'YesNoVar', + NO_REGRESS => 'YesNoVar', + NOT_FOR_ARCHS => 'NotForArchListVar', + ONLY_FOR_ARCHS => 'OnlyForArchListVar', + PERMIT_DISTFILES_CDROM => 'YesKeyVar', + PERMIT_DISTFILES_FTP=> 'YesKeyVar', + PERMIT_PACKAGE_CDROM => 'YesKeyVar', + PERMIT_PACKAGE_FTP=> 'YesKeyVar', + PKGNAME => 'AnyVar', + PKGSPEC => 'AnyVar', + PKG_ARCH => 'ArchKeyVar', + PSEUDO_FLAVOR => 'AnyVar', + PSEUDO_FLAVORS => 'PseudoFlavorsVar', + REGRESS_DEPENDS => 'RegressDependsVar', + REGRESS_IS_INTERACTIVE => 'AnyVar', + REVISION => 'AnyVar', + RUN_DEPENDS => 'RunDependsVar', + SEPARATE_BUILD => 'YesKeyVar', + SHARED_LIBS => 'SharedLibsVar', + SHARED_ONLY => 'YesNoVar', + SUBPACKAGE => 'DefinedVar', + SUPDISTFILES => 'AnyVar', + TARGETS => 'TargetsVar', + USE_GMAKE => 'YesNoVar', + USE_GROFF => 'YesNoVar', + USE_LIBTOOL => 'YesNoGnuVar', + VMEM_WARNING => 'YesNoVar', + WANTLIB => 'WantlibVar', +}; + +our $unknown = {}; + +sub new +{ + my ($class, $p) = @_; + bless {path => $p, vars => {}}, $class; +} + +sub create +{ + my ($self, $var, $value, $arch) = @_; + my $k = $var; + if (defined $arch) { + $k .= "-$arch"; + } + if (defined $vars->{$var}) { + $self->{vars}{$k} = $vars->{$var}->new($var, $value, $arch); + } else { + $unknown->{$k} //= $self->{path}; + } +} + +sub variables +{ + my $self = shift; + return values %{$self->{vars}}; +} + +sub value +{ + my ($self, $name) = @_; + return $self->{vars}{$name}->value; +} + +sub reclaim +{ + my $self = shift; + my $n = {}; + for my $k (qw(SUBPACKAGE FLAVOR)) { + $n->{$k} = $self->{vars}{$k}; + } + $self->{vars} = $n; +} + +1; diff --git a/databases/sqlports/files/Inserter.pm b/databases/sqlports/files/Inserter.pm index dc3832c1967..a9977d20dbe 100644 --- a/databases/sqlports/files/Inserter.pm +++ b/databases/sqlports/files/Inserter.pm @@ -1,5 +1,5 @@ #! /usr/bin/perl -# $OpenBSD: Inserter.pm,v 1.9 2011/03/02 16:19:54 espie Exp $ +# $OpenBSD: Inserter.pm,v 1.10 2012/05/18 12:11:28 espie Exp $ # # Copyright (c) 2006-2010 Marc Espie # @@ -89,14 +89,6 @@ sub add_error { } -sub add_todo -{ -} - -sub mark_done -{ -} - sub write_log { } @@ -107,8 +99,7 @@ sub create_tables $self->create_path_table; while (my ($name, $varclass) = each %$vars) { - $self->handle_column($varclass->column($name)); - $varclass->create_table($self); + $varclass->prepare_tables($self, $name); } $self->create_ports_table; @@ -439,18 +430,6 @@ sub add_error push(@{$self->{errors}}, $msg); } -sub add_todo -{ - my ($self, $path) = @_; - $self->{todo}{$path} = 1; -} - -sub mark_done -{ - my ($self, $path) = @_; - $self->{done}{$path} = 1; -} - sub write_log { my ($self, $log) = @_; diff --git a/databases/sqlports/files/PkgPath.pm b/databases/sqlports/files/PkgPath.pm new file mode 100644 index 00000000000..d333f495926 --- /dev/null +++ b/databases/sqlports/files/PkgPath.pm @@ -0,0 +1,66 @@ +# ex:ts=8 sw=4: +# $OpenBSD: PkgPath.pm,v 1.1 2012/05/18 12:11:28 espie Exp $ +# +# Copyright (c) 2012 Marc Espie +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +BEGIN { + $ports1 = $ENV{PORTSDIR} || '/usr/ports'; +} +use lib ("$ports1/infrastructure/lib"); + +use DPB::BasePkgPath; + +package PkgPath; +our @ISA = qw(DPB::BasePkgPath); +use Info; + +sub init +{ +} + +sub clone_properties +{ + my ($n, $o) = @_; + $n->{info} //= $o->{info}; +} + +sub subpackage +{ + my $self = shift; + return $self->{info}->value('SUBPACKAGE'); +} + +sub flavor +{ + my $self = shift; + my $value = $self->{info}->value('FLAVOR'); + $value =~ s/^\s+//; + $value =~ s/\s+$//; + my @l = split(/\s+/, $value); + my %values = map {($_,1)} @l; + + return \%values; +} + +sub equates +{ +} + +sub simplifies_to +{ + my ($self, $other, $inserter) = @_; +} + +1; diff --git a/databases/sqlports/files/Var.pm b/databases/sqlports/files/Var.pm index f3be8f758f1..27c3959ab81 100644 --- a/databases/sqlports/files/Var.pm +++ b/databases/sqlports/files/Var.pm @@ -1,4 +1,4 @@ -# $OpenBSD: Var.pm,v 1.12 2011/11/26 22:30:30 kili Exp $ +# $OpenBSD: Var.pm,v 1.13 2012/05/18 12:11:28 espie Exp $ # # Copyright (c) 2006-2010 Marc Espie # @@ -72,6 +72,13 @@ sub column return $self->columntype->new($name)->set_vartype($self); } +sub prepare_tables +{ + my ($self, $inserter, $name) = @_; + $inserter->handle_column($self->column($name)); + $self->create_tables($inserter); +} + sub keyword { my ($self, $ins, $value) = @_; @@ -86,7 +93,7 @@ sub create_keyword_table } } -sub create_table +sub create_tables { my ($self, $inserter) = @_; $self->create_keyword_table($inserter); @@ -161,7 +168,7 @@ sub add $self->normal_insert($ins, $arch, $self->value); } -sub create_table +sub create_tables { my ($self, $inserter) = @_; $self->create_keyword_table($inserter); @@ -260,15 +267,17 @@ sub add print STDERR "Wrong depends $depends\n"; return; } + my $p = PkgPath->new($pkgpath2); + $p->{want} = 1; $self->normal_insert($ins, $depends, $ins->find_pathkey($pkgpath2), $ins->convert_depends($self->depends_type), $pkgspec, $rest); - $ins->add_todo($pkgpath2); +# XXX $ins->add_todo($pkgpath2); } } -sub create_table +sub create_tables { my ($self, $inserter) = @_; $inserter->make_table($self, undef, @@ -314,7 +323,7 @@ sub add_keyword $self->add_value($ins, $self->keyword($ins, $value)); } -sub create_table +sub create_tables { my ($self, $inserter) = @_; $self->create_keyword_table($inserter); @@ -339,7 +348,7 @@ sub add $self->normal_insert($ins, $n, $self->value); } -sub create_table +sub create_tables { my ($self, $inserter) = @_; $self->create_keyword_table($inserter); @@ -487,7 +496,7 @@ sub add_value } } -sub create_table +sub create_tables { my ($self, $inserter) = @_; $self->create_keyword_table($inserter); @@ -530,7 +539,7 @@ sub add } } -sub create_table +sub create_tables { my ($self, $inserter) = @_; $self->create_keyword_table($inserter); @@ -552,4 +561,15 @@ package AutoVersionVar; our @ISA = qw(OptKeyVar); sub keyword_table() { 'AutoVersion' } +package IgnoredVar; +our @ISA = qw(AnyVar); + +sub add +{ +} + +sub prepare_tables +{ +} + 1; diff --git a/databases/sqlports/files/mksqlitedb b/databases/sqlports/files/mksqlitedb index 706014f5d7f..9e03e794106 100644 --- a/databases/sqlports/files/mksqlitedb +++ b/databases/sqlports/files/mksqlitedb @@ -1,5 +1,5 @@ #! /usr/bin/perl -# $OpenBSD: mksqlitedb,v 1.37 2012/05/08 17:22:00 espie Exp $ +# $OpenBSD: mksqlitedb,v 1.38 2012/05/18 12:11:28 espie Exp $ # # Copyright (c) 2006-2010 Marc Espie # @@ -29,64 +29,90 @@ use Column; use Var; use Inserter; use DBI; - +use PkgPath; +use Info; our ($opt_v, $opt_q, $opt_p); +sub subdirlist +{ + my $list = shift; + return join(' ', sort keys %$list); +} + sub parse_dump { - my ($inserter, $vars, $fd, $unknown) = @_; - my $lastkey; + my ($inserter, $fd, $subdirs) = @_; + my $h = {}; + my $seen = {}; + my $subdir; + my $reset = sub { + $h = PkgPath->handle_equivalences($inserter, $h, $subdirs); + for my $pkgpath (values %$h) { + my $key = $pkgpath->fullpkgpath; + if ($pkgpath->{info}{done}) { + print "--- $key (already done)\n"; + next; + } + print "+++ $key\n"; + $inserter->set_newkey($key); + for my $var ($pkgpath->{info}->variables) { + $inserter->add_var($var); + } + $pkgpath->{info}->reclaim; + $pkgpath->{info}{done} = 1; + $inserter->finish_port; + } + $h = {}; + }; + while (<$fd>) { chomp; # kill noise - if (s/^\=\=\=/---/) { - print $_, "\n"; + if (m/^\=\=\=\>\s*Exiting (.*) with an error$/) { + my $dir = PkgPath->new($1); + $dir->break("exiting with an error"); + $h->{$dir} = $dir; + &$reset; next; } - next unless m/^(.*?)\.([A-Z][A-Z_0-9]*)(?:\-([a-z0-9]+))?\=(.*)$/; - - my ($key, $var, $arch, $value) = ($1, $2, $3, $4); - # strip extra quotes - if ($value =~ m/^\"(.*)\"$/) { - $value = $1; - } - - if (!(defined $lastkey) || $key ne $lastkey) { - if (defined $lastkey) { - $inserter->finish_port; + if (m/^\=\=\=\>\s*(.*)/) { + $subdir = PkgPath->new($1); + &$reset; + } elsif (my ($pkgpath, $var, $arch, $value) = + m/^(.*?)\.([A-Z][A-Z_0-9]*)(?:\-([a-z0-9]+))?\=\s*(.*)\s*$/) { + if ($value =~ m/^\"(.*)\"$/) { + $value = $1; } - $inserter->mark_done($key); - if ($key =~ m/^(.*)\,\-main((?:\,.*)?)$/) { - $inserter->mark_done("$1$2"); - } - $inserter->set_newkey($key); - $lastkey = $key; - } - if (defined $vars->{$var}) { - $inserter->add_var($vars->{$var}->new($var, $value, $arch)); - } else { - $unknown->{$var} //= $key; + my $o = PkgPath->compose($pkgpath, $subdir); + $h->{$o} = $o; + # Note we did it ! + $o->{info} //= Info->new($o); + $o->{info}->create($var, $value, $arch); + } elsif (m/^\>\>\s*Broken dependency:\s*(.*?)\s*non existent/) { + my $dir = PkgPath->new($1); + $dir->break("broken dependency"); + $h->{$dir} = $dir; + &$reset; } } - - $inserter->finish_port; + &$reset; $inserter->commit_to_db; } sub dump_dirs { - my ($inserter, $vars, $unknown, $todo) = @_; + my ($inserter, $subdirs) = @_; my ($pid, $fd); unless (defined($pid = open($fd, "-|"))) { die "can't fork : $!"; } if ($pid) { - parse_dump($inserter, $vars, $fd, $unknown); + parse_dump($inserter, $fd, $subdirs); close $fd || die $!; } else { - if (defined $todo) { - $ENV{'SUBDIR'} = join(' ', keys %$todo); + if (defined $subdirs) { + $ENV{'SUBDIR'} = subdirlist($subdirs); } if ($opt_p) { $ENV{'REPORT_PROBLEM_LOGFILE'}= $opt_p; @@ -116,103 +142,34 @@ my $normal = NormalInserter->new($db, 1000, $opt_v); $inserter->add($normal, CompactInserter->new($db2, 1000, $opt_v)); -my $vars = { - AUTOCONF_VERSION => 'AutoVersionVar', - AUTOMAKE_VERSION => 'AutoVersionVar', - BROKEN => 'BrokenVar', - BUILD_DEPENDS => 'BuildDependsVar', - CATEGORIES => 'CategoriesVar', - COMES_WITH => 'DefinedVar', - COMMENT => 'AnyVar', - CONFIGURE_ARGS => 'ConfigureArgsVar', - CONFIGURE_STYLE => 'ConfigureVar', - DESCR => 'FileVar', - DISTFILES => 'AnyVar', - PATCHFILES => 'AnyVar', - DISTNAME => 'AnyVar', - DIST_SUBDIR => 'DefinedVar', - EPOCH => 'AnyVar', - FLAVORS => 'FlavorsVar', - FULLPKGNAME => 'AnyVar', - HOMEPAGE => 'AnyVar', - IGNORE => 'DefinedVar', - IS_INTERACTIVE => 'AnyVar', - LIB_DEPENDS => 'LibDependsVar', - MAINTAINER=> 'EmailVar', - MASTER_SITES => 'MasterSitesVar', - MASTER_SITES0 => 'MasterSitesVar', - MASTER_SITES1 => 'MasterSitesVar', - MASTER_SITES2 => 'MasterSitesVar', - MASTER_SITES3 => 'MasterSitesVar', - MASTER_SITES4=> 'MasterSitesVar', - MASTER_SITES5 => 'MasterSitesVar', - MASTER_SITES6 => 'MasterSitesVar', - MASTER_SITES7 => 'MasterSitesVar', - MASTER_SITES8 => 'MasterSitesVar', - MASTER_SITES9=> 'MasterSitesVar', - MODULES => 'ModulesVar', - MULTI_PACKAGES => 'MultiVar', - NO_BUILD => 'YesNoVar', - NO_REGRESS => 'YesNoVar', - NOT_FOR_ARCHS => 'NotForArchListVar', - ONLY_FOR_ARCHS => 'OnlyForArchListVar', - PERMIT_DISTFILES_CDROM => 'YesKeyVar', - PERMIT_DISTFILES_FTP=> 'YesKeyVar', - PERMIT_PACKAGE_CDROM => 'YesKeyVar', - PERMIT_PACKAGE_FTP=> 'YesKeyVar', - PKGNAME => 'AnyVar', - PKGSPEC => 'AnyVar', - PKG_ARCH => 'ArchKeyVar', - PSEUDO_FLAVOR => 'AnyVar', - PSEUDO_FLAVORS => 'PseudoFlavorsVar', - REGRESS_DEPENDS => 'RegressDependsVar', - REGRESS_IS_INTERACTIVE => 'AnyVar', - REVISION => 'AnyVar', - RUN_DEPENDS => 'RunDependsVar', - SEPARATE_BUILD => 'YesKeyVar', - SHARED_LIBS => 'SharedLibsVar', - SHARED_ONLY => 'YesNoVar', - SUBPACKAGE => 'DefinedVar', - SUPDISTFILES => 'AnyVar', - TARGETS => 'TargetsVar', - USE_GMAKE => 'YesNoVar', - USE_GROFF => 'YesNoVar', - USE_LIBTOOL => 'YesNoGnuVar', - VMEM_WARNING => 'YesNoVar', - WANTLIB => 'WantlibVar', -}; +$inserter->create_tables($Info::vars); -$inserter->create_tables($vars); - -my $unknown = {}; - -dump_dirs($inserter, $vars, $unknown, undef); +dump_dirs($inserter, undef); my $i = 1; while (1) { - my $todo = {}; - my $stuff = 0; - for my $v (keys %{$normal->{todo}}) { - next if $normal->{done}{$v}; - $todo->{$v} = 1; - $stuff = 1; - - } - last if !$stuff; - $normal->{todo} = {}; - $i++; - print "pass #$i\n"; - dump_dirs($inserter, $vars, $unknown, $todo); - for my $v (keys %$todo) { - if (!$normal->{done}{$v}) { - $normal->{done}{$v} = 1; + my $subdirlist = {}; + for my $v (PkgPath->seen) { + if (defined $v->{info}) { + delete $v->{tried}; + delete $v->{want}; + next; + } + if (defined $v->{tried}) { + } elsif ($v->{want}) { + $v->add_to_subdirlist($subdirlist); + $v->{tried} = 1; } } + last if (keys %$subdirlist) == 0; + $i++; + print "pass #$i\n"; + dump_dirs($inserter, $subdirlist); } -while (my ($k, $v) = each %$unknown) { +while (my ($k, $v) = each %$Info::unknown) { next if $k eq 'CHECKSUM_FILE'; - print STDERR "Unknown variable $k in $v\n"; + print STDERR "Unknown variable $k in ", $v->fullpkgpath, "\n"; } if (defined $opt_q) {