bite the bullet, reuse dpb's algorithm to make sure we get correct

PkgPath.

This implies restructuring the database code, and the start of a
better separation between variables handling, and actual insertion
in the database.

bump version to 2.0: this changes the paths that actually get recorded,
and pkglocatedb will need to be aware of that.
This commit is contained in:
espie 2012-05-18 12:11:28 +00:00
parent 9aa5ebca2b
commit cea1d3a0f3
6 changed files with 319 additions and 157 deletions

View File

@ -1,7 +1,7 @@
# $OpenBSD: Makefile,v 1.39 2012/04/29 16:34:53 espie Exp $
# $OpenBSD: Makefile,v 1.40 2012/05/18 12:11:28 espie Exp $
CATEGORIES = databases
V = 1.20
V = 2.0
DISTNAME = sqlports-$V
DISTFILES =
COMMENT = sqlite database of ports
@ -21,7 +21,7 @@ MULTI_PACKAGES = -main -compact
DBNAME = ${WRKBUILD}/sqlports
do-build:
@cd ${PORTSDIR} && perl ${FILESDIR}/mksqlitedb -v ${DBNAME} -p ${WRKBUILD}/ouch
@cd ${PORTSDIR} && PORTSDIR=${PORTSDIR} perl ${FILESDIR}/mksqlitedb -v ${DBNAME} -p ${WRKBUILD}/ouch
@if test -s ${WRKBUILD}/ouch; then \
cat ${WRKBUILD}/ouch; \
exit 1; \

View File

@ -0,0 +1,140 @@
# $OpenBSD: Info.pm,v 1.1 2012/05/18 12:11:28 espie Exp $
#
# Copyright (c) 2012 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# example script that shows how to store all variable values into a
# database, using SQLite for that purpose.
#
# usage: cd /usr/ports && make dump-vars |mksqlitedb
use strict;
use warnings;
use Var;
package Info;
our $vars = {
AUTOCONF_VERSION => 'AutoVersionVar',
AUTOMAKE_VERSION => 'AutoVersionVar',
BROKEN => 'BrokenVar',
BUILD_DEPENDS => 'BuildDependsVar',
CATEGORIES => 'CategoriesVar',
COMES_WITH => 'DefinedVar',
COMMENT => 'AnyVar',
CONFIGURE_ARGS => 'ConfigureArgsVar',
CONFIGURE_STYLE => 'ConfigureVar',
DESCR => 'FileVar',
DISTFILES => 'AnyVar',
PATCHFILES => 'AnyVar',
DISTNAME => 'AnyVar',
DIST_SUBDIR => 'DefinedVar',
EPOCH => 'AnyVar',
FETCH_MANUALLY => 'IgnoredVar',
FLAVOR => 'IgnoredVar',
FLAVORS => 'FlavorsVar',
FULLPKGNAME => 'AnyVar',
HOMEPAGE => 'AnyVar',
IGNORE => 'DefinedVar',
IS_INTERACTIVE => 'AnyVar',
LIB_DEPENDS => 'LibDependsVar',
MAINTAINER=> 'EmailVar',
MASTER_SITES => 'MasterSitesVar',
MASTER_SITES0 => 'MasterSitesVar',
MASTER_SITES1 => 'MasterSitesVar',
MASTER_SITES2 => 'MasterSitesVar',
MASTER_SITES3 => 'MasterSitesVar',
MASTER_SITES4=> 'MasterSitesVar',
MASTER_SITES5 => 'MasterSitesVar',
MASTER_SITES6 => 'MasterSitesVar',
MASTER_SITES7 => 'MasterSitesVar',
MASTER_SITES8 => 'MasterSitesVar',
MASTER_SITES9=> 'MasterSitesVar',
MISSING_FILES => 'IgnoredVar',
MODULES => 'ModulesVar',
MULTI_PACKAGES => 'MultiVar',
NO_BUILD => 'YesNoVar',
NO_REGRESS => 'YesNoVar',
NOT_FOR_ARCHS => 'NotForArchListVar',
ONLY_FOR_ARCHS => 'OnlyForArchListVar',
PERMIT_DISTFILES_CDROM => 'YesKeyVar',
PERMIT_DISTFILES_FTP=> 'YesKeyVar',
PERMIT_PACKAGE_CDROM => 'YesKeyVar',
PERMIT_PACKAGE_FTP=> 'YesKeyVar',
PKGNAME => 'AnyVar',
PKGSPEC => 'AnyVar',
PKG_ARCH => 'ArchKeyVar',
PSEUDO_FLAVOR => 'AnyVar',
PSEUDO_FLAVORS => 'PseudoFlavorsVar',
REGRESS_DEPENDS => 'RegressDependsVar',
REGRESS_IS_INTERACTIVE => 'AnyVar',
REVISION => 'AnyVar',
RUN_DEPENDS => 'RunDependsVar',
SEPARATE_BUILD => 'YesKeyVar',
SHARED_LIBS => 'SharedLibsVar',
SHARED_ONLY => 'YesNoVar',
SUBPACKAGE => 'DefinedVar',
SUPDISTFILES => 'AnyVar',
TARGETS => 'TargetsVar',
USE_GMAKE => 'YesNoVar',
USE_GROFF => 'YesNoVar',
USE_LIBTOOL => 'YesNoGnuVar',
VMEM_WARNING => 'YesNoVar',
WANTLIB => 'WantlibVar',
};
our $unknown = {};
sub new
{
my ($class, $p) = @_;
bless {path => $p, vars => {}}, $class;
}
sub create
{
my ($self, $var, $value, $arch) = @_;
my $k = $var;
if (defined $arch) {
$k .= "-$arch";
}
if (defined $vars->{$var}) {
$self->{vars}{$k} = $vars->{$var}->new($var, $value, $arch);
} else {
$unknown->{$k} //= $self->{path};
}
}
sub variables
{
my $self = shift;
return values %{$self->{vars}};
}
sub value
{
my ($self, $name) = @_;
return $self->{vars}{$name}->value;
}
sub reclaim
{
my $self = shift;
my $n = {};
for my $k (qw(SUBPACKAGE FLAVOR)) {
$n->{$k} = $self->{vars}{$k};
}
$self->{vars} = $n;
}
1;

View File

@ -1,5 +1,5 @@
#! /usr/bin/perl
# $OpenBSD: Inserter.pm,v 1.9 2011/03/02 16:19:54 espie Exp $
# $OpenBSD: Inserter.pm,v 1.10 2012/05/18 12:11:28 espie Exp $
#
# Copyright (c) 2006-2010 Marc Espie <espie@openbsd.org>
#
@ -89,14 +89,6 @@ sub add_error
{
}
sub add_todo
{
}
sub mark_done
{
}
sub write_log
{
}
@ -107,8 +99,7 @@ sub create_tables
$self->create_path_table;
while (my ($name, $varclass) = each %$vars) {
$self->handle_column($varclass->column($name));
$varclass->create_table($self);
$varclass->prepare_tables($self, $name);
}
$self->create_ports_table;
@ -439,18 +430,6 @@ sub add_error
push(@{$self->{errors}}, $msg);
}
sub add_todo
{
my ($self, $path) = @_;
$self->{todo}{$path} = 1;
}
sub mark_done
{
my ($self, $path) = @_;
$self->{done}{$path} = 1;
}
sub write_log
{
my ($self, $log) = @_;

View File

@ -0,0 +1,66 @@
# ex:ts=8 sw=4:
# $OpenBSD: PkgPath.pm,v 1.1 2012/05/18 12:11:28 espie Exp $
#
# Copyright (c) 2012 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
BEGIN {
$ports1 = $ENV{PORTSDIR} || '/usr/ports';
}
use lib ("$ports1/infrastructure/lib");
use DPB::BasePkgPath;
package PkgPath;
our @ISA = qw(DPB::BasePkgPath);
use Info;
sub init
{
}
sub clone_properties
{
my ($n, $o) = @_;
$n->{info} //= $o->{info};
}
sub subpackage
{
my $self = shift;
return $self->{info}->value('SUBPACKAGE');
}
sub flavor
{
my $self = shift;
my $value = $self->{info}->value('FLAVOR');
$value =~ s/^\s+//;
$value =~ s/\s+$//;
my @l = split(/\s+/, $value);
my %values = map {($_,1)} @l;
return \%values;
}
sub equates
{
}
sub simplifies_to
{
my ($self, $other, $inserter) = @_;
}
1;

View File

@ -1,4 +1,4 @@
# $OpenBSD: Var.pm,v 1.12 2011/11/26 22:30:30 kili Exp $
# $OpenBSD: Var.pm,v 1.13 2012/05/18 12:11:28 espie Exp $
#
# Copyright (c) 2006-2010 Marc Espie <espie@openbsd.org>
#
@ -72,6 +72,13 @@ sub column
return $self->columntype->new($name)->set_vartype($self);
}
sub prepare_tables
{
my ($self, $inserter, $name) = @_;
$inserter->handle_column($self->column($name));
$self->create_tables($inserter);
}
sub keyword
{
my ($self, $ins, $value) = @_;
@ -86,7 +93,7 @@ sub create_keyword_table
}
}
sub create_table
sub create_tables
{
my ($self, $inserter) = @_;
$self->create_keyword_table($inserter);
@ -161,7 +168,7 @@ sub add
$self->normal_insert($ins, $arch, $self->value);
}
sub create_table
sub create_tables
{
my ($self, $inserter) = @_;
$self->create_keyword_table($inserter);
@ -260,15 +267,17 @@ sub add
print STDERR "Wrong depends $depends\n";
return;
}
my $p = PkgPath->new($pkgpath2);
$p->{want} = 1;
$self->normal_insert($ins, $depends,
$ins->find_pathkey($pkgpath2),
$ins->convert_depends($self->depends_type),
$pkgspec, $rest);
$ins->add_todo($pkgpath2);
# XXX $ins->add_todo($pkgpath2);
}
}
sub create_table
sub create_tables
{
my ($self, $inserter) = @_;
$inserter->make_table($self, undef,
@ -314,7 +323,7 @@ sub add_keyword
$self->add_value($ins, $self->keyword($ins, $value));
}
sub create_table
sub create_tables
{
my ($self, $inserter) = @_;
$self->create_keyword_table($inserter);
@ -339,7 +348,7 @@ sub add
$self->normal_insert($ins, $n, $self->value);
}
sub create_table
sub create_tables
{
my ($self, $inserter) = @_;
$self->create_keyword_table($inserter);
@ -487,7 +496,7 @@ sub add_value
}
}
sub create_table
sub create_tables
{
my ($self, $inserter) = @_;
$self->create_keyword_table($inserter);
@ -530,7 +539,7 @@ sub add
}
}
sub create_table
sub create_tables
{
my ($self, $inserter) = @_;
$self->create_keyword_table($inserter);
@ -552,4 +561,15 @@ package AutoVersionVar;
our @ISA = qw(OptKeyVar);
sub keyword_table() { 'AutoVersion' }
package IgnoredVar;
our @ISA = qw(AnyVar);
sub add
{
}
sub prepare_tables
{
}
1;

View File

@ -1,5 +1,5 @@
#! /usr/bin/perl
# $OpenBSD: mksqlitedb,v 1.37 2012/05/08 17:22:00 espie Exp $
# $OpenBSD: mksqlitedb,v 1.38 2012/05/18 12:11:28 espie Exp $
#
# Copyright (c) 2006-2010 Marc Espie <espie@openbsd.org>
#
@ -29,64 +29,90 @@ use Column;
use Var;
use Inserter;
use DBI;
use PkgPath;
use Info;
our ($opt_v, $opt_q, $opt_p);
sub subdirlist
{
my $list = shift;
return join(' ', sort keys %$list);
}
sub parse_dump
{
my ($inserter, $vars, $fd, $unknown) = @_;
my $lastkey;
my ($inserter, $fd, $subdirs) = @_;
my $h = {};
my $seen = {};
my $subdir;
my $reset = sub {
$h = PkgPath->handle_equivalences($inserter, $h, $subdirs);
for my $pkgpath (values %$h) {
my $key = $pkgpath->fullpkgpath;
if ($pkgpath->{info}{done}) {
print "--- $key (already done)\n";
next;
}
print "+++ $key\n";
$inserter->set_newkey($key);
for my $var ($pkgpath->{info}->variables) {
$inserter->add_var($var);
}
$pkgpath->{info}->reclaim;
$pkgpath->{info}{done} = 1;
$inserter->finish_port;
}
$h = {};
};
while (<$fd>) {
chomp;
# kill noise
if (s/^\=\=\=/---/) {
print $_, "\n";
if (m/^\=\=\=\>\s*Exiting (.*) with an error$/) {
my $dir = PkgPath->new($1);
$dir->break("exiting with an error");
$h->{$dir} = $dir;
&$reset;
next;
}
next unless m/^(.*?)\.([A-Z][A-Z_0-9]*)(?:\-([a-z0-9]+))?\=(.*)$/;
my ($key, $var, $arch, $value) = ($1, $2, $3, $4);
# strip extra quotes
if ($value =~ m/^\"(.*)\"$/) {
$value = $1;
}
if (!(defined $lastkey) || $key ne $lastkey) {
if (defined $lastkey) {
$inserter->finish_port;
if (m/^\=\=\=\>\s*(.*)/) {
$subdir = PkgPath->new($1);
&$reset;
} elsif (my ($pkgpath, $var, $arch, $value) =
m/^(.*?)\.([A-Z][A-Z_0-9]*)(?:\-([a-z0-9]+))?\=\s*(.*)\s*$/) {
if ($value =~ m/^\"(.*)\"$/) {
$value = $1;
}
$inserter->mark_done($key);
if ($key =~ m/^(.*)\,\-main((?:\,.*)?)$/) {
$inserter->mark_done("$1$2");
}
$inserter->set_newkey($key);
$lastkey = $key;
}
if (defined $vars->{$var}) {
$inserter->add_var($vars->{$var}->new($var, $value, $arch));
} else {
$unknown->{$var} //= $key;
my $o = PkgPath->compose($pkgpath, $subdir);
$h->{$o} = $o;
# Note we did it !
$o->{info} //= Info->new($o);
$o->{info}->create($var, $value, $arch);
} elsif (m/^\>\>\s*Broken dependency:\s*(.*?)\s*non existent/) {
my $dir = PkgPath->new($1);
$dir->break("broken dependency");
$h->{$dir} = $dir;
&$reset;
}
}
$inserter->finish_port;
&$reset;
$inserter->commit_to_db;
}
sub dump_dirs
{
my ($inserter, $vars, $unknown, $todo) = @_;
my ($inserter, $subdirs) = @_;
my ($pid, $fd);
unless (defined($pid = open($fd, "-|"))) {
die "can't fork : $!";
}
if ($pid) {
parse_dump($inserter, $vars, $fd, $unknown);
parse_dump($inserter, $fd, $subdirs);
close $fd || die $!;
} else {
if (defined $todo) {
$ENV{'SUBDIR'} = join(' ', keys %$todo);
if (defined $subdirs) {
$ENV{'SUBDIR'} = subdirlist($subdirs);
}
if ($opt_p) {
$ENV{'REPORT_PROBLEM_LOGFILE'}= $opt_p;
@ -116,103 +142,34 @@ my $normal = NormalInserter->new($db, 1000, $opt_v);
$inserter->add($normal, CompactInserter->new($db2, 1000, $opt_v));
my $vars = {
AUTOCONF_VERSION => 'AutoVersionVar',
AUTOMAKE_VERSION => 'AutoVersionVar',
BROKEN => 'BrokenVar',
BUILD_DEPENDS => 'BuildDependsVar',
CATEGORIES => 'CategoriesVar',
COMES_WITH => 'DefinedVar',
COMMENT => 'AnyVar',
CONFIGURE_ARGS => 'ConfigureArgsVar',
CONFIGURE_STYLE => 'ConfigureVar',
DESCR => 'FileVar',
DISTFILES => 'AnyVar',
PATCHFILES => 'AnyVar',
DISTNAME => 'AnyVar',
DIST_SUBDIR => 'DefinedVar',
EPOCH => 'AnyVar',
FLAVORS => 'FlavorsVar',
FULLPKGNAME => 'AnyVar',
HOMEPAGE => 'AnyVar',
IGNORE => 'DefinedVar',
IS_INTERACTIVE => 'AnyVar',
LIB_DEPENDS => 'LibDependsVar',
MAINTAINER=> 'EmailVar',
MASTER_SITES => 'MasterSitesVar',
MASTER_SITES0 => 'MasterSitesVar',
MASTER_SITES1 => 'MasterSitesVar',
MASTER_SITES2 => 'MasterSitesVar',
MASTER_SITES3 => 'MasterSitesVar',
MASTER_SITES4=> 'MasterSitesVar',
MASTER_SITES5 => 'MasterSitesVar',
MASTER_SITES6 => 'MasterSitesVar',
MASTER_SITES7 => 'MasterSitesVar',
MASTER_SITES8 => 'MasterSitesVar',
MASTER_SITES9=> 'MasterSitesVar',
MODULES => 'ModulesVar',
MULTI_PACKAGES => 'MultiVar',
NO_BUILD => 'YesNoVar',
NO_REGRESS => 'YesNoVar',
NOT_FOR_ARCHS => 'NotForArchListVar',
ONLY_FOR_ARCHS => 'OnlyForArchListVar',
PERMIT_DISTFILES_CDROM => 'YesKeyVar',
PERMIT_DISTFILES_FTP=> 'YesKeyVar',
PERMIT_PACKAGE_CDROM => 'YesKeyVar',
PERMIT_PACKAGE_FTP=> 'YesKeyVar',
PKGNAME => 'AnyVar',
PKGSPEC => 'AnyVar',
PKG_ARCH => 'ArchKeyVar',
PSEUDO_FLAVOR => 'AnyVar',
PSEUDO_FLAVORS => 'PseudoFlavorsVar',
REGRESS_DEPENDS => 'RegressDependsVar',
REGRESS_IS_INTERACTIVE => 'AnyVar',
REVISION => 'AnyVar',
RUN_DEPENDS => 'RunDependsVar',
SEPARATE_BUILD => 'YesKeyVar',
SHARED_LIBS => 'SharedLibsVar',
SHARED_ONLY => 'YesNoVar',
SUBPACKAGE => 'DefinedVar',
SUPDISTFILES => 'AnyVar',
TARGETS => 'TargetsVar',
USE_GMAKE => 'YesNoVar',
USE_GROFF => 'YesNoVar',
USE_LIBTOOL => 'YesNoGnuVar',
VMEM_WARNING => 'YesNoVar',
WANTLIB => 'WantlibVar',
};
$inserter->create_tables($Info::vars);
$inserter->create_tables($vars);
my $unknown = {};
dump_dirs($inserter, $vars, $unknown, undef);
dump_dirs($inserter, undef);
my $i = 1;
while (1) {
my $todo = {};
my $stuff = 0;
for my $v (keys %{$normal->{todo}}) {
next if $normal->{done}{$v};
$todo->{$v} = 1;
$stuff = 1;
}
last if !$stuff;
$normal->{todo} = {};
$i++;
print "pass #$i\n";
dump_dirs($inserter, $vars, $unknown, $todo);
for my $v (keys %$todo) {
if (!$normal->{done}{$v}) {
$normal->{done}{$v} = 1;
my $subdirlist = {};
for my $v (PkgPath->seen) {
if (defined $v->{info}) {
delete $v->{tried};
delete $v->{want};
next;
}
if (defined $v->{tried}) {
} elsif ($v->{want}) {
$v->add_to_subdirlist($subdirlist);
$v->{tried} = 1;
}
}
last if (keys %$subdirlist) == 0;
$i++;
print "pass #$i\n";
dump_dirs($inserter, $subdirlist);
}
while (my ($k, $v) = each %$unknown) {
while (my ($k, $v) = each %$Info::unknown) {
next if $k eq 'CHECKSUM_FILE';
print STDERR "Unknown variable $k in $v\n";
print STDERR "Unknown variable $k in ", $v->fullpkgpath, "\n";
}
if (defined $opt_q) {