openbsd-ports/databases/sqlports/files/mksqlitedb

#! /usr/bin/perl
# $OpenBSD: mksqlitedb,v 1.4 2006/12/27 11:16:10 steven Exp $
#
# Copyright (c) 2006 Marc Espie <espie@openbsd.org>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

# example script that shows how to store all variable values into a
# database, using SQLite for that purpose.
#
# usage: cd /usr/ports && make dump-vars |mksqlitedb

use strict;
use warnings;

sub words($)
{
	my $v = shift;
	$v =~ s/^\s+//;
	$v =~ s/\s+$//;
	return split(/\s+/, $v);
}

# use a Template Method approach to store the variable values.

# rule: we store each value in the main table, after converting YesNo
# variables to undef/1. Then, in addition, we process specific variables
# to store them in secondary tables (because of one/many associations).

package AnyVar;
sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	my $stmt=$db->prepare("UPDATE Ports SET $var=(?) WHERE RowID=(?)");
	$stmt->execute($value, $rowid);
}

package YesNoVar;
our @ISA=(qw(AnyVar));

sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	$class->SUPER::add($pkgpath, $db, $var, $value =~ m/^Yes/i ? 1 : undef, $rowid);
}

# variable is always defined, but we don't need to store empty values.
package DefinedVar;
our @ISA=(qw(AnyVar));

sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	return if $value eq '';
	$class->SUPER::add($pkgpath, $db, $var, $value, $rowid);
}


# all the dependencies are converted into list. Stuff like LIB_DEPENDS will
# end up being treated as WANTLIB as well.

package DependsVar;
sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	AnyVar->add($pkgpath, $db, $var, $value, $rowid);
	for my $depends (main::words $value) {
		my ($libs, $pkgspec, $pkgpath2, $rest) = split(/\:/, $depends);
		my $stmt = $db->prepare("INSERT INTO Depends (FULLPKGPATH, FULLDEPENDS, DEPENDSPATH, TYPE) VALUES (?, ?, ?, ?)");
		$stmt->execute($pkgpath, $depends, $pkgpath2, $class->type());
		if ($libs ne '') {
			for my $lib (split(/\,/, $libs)) {
				$class->addlib($pkgpath, $db, $lib);
			}
		}
	}
}

sub addlib
{
}

package LibDependsVar;
our @ISA=(qw(DependsVar));
sub type() { 'L' }

sub addlib
{
	my ($class, $pkgpath, $db, $lib) = @_;
	WantlibVar->addvalue($pkgpath, $db, $lib);
}

package RunDependsVar;
our @ISA=(qw(DependsVar));
sub type() { 'R' }

package BuildDependsVar;
our @ISA=(qw(DependsVar));
sub type() { 'B' }

package RegressDependsVar;
our @ISA=(qw(DependsVar));
sub type() { 'Regress' }

# Stuff that gets stored in another table as well
package SecondaryVar;
sub addvalue
{
	my ($class, $pkgpath, $db, $value) = @_;
	my $stmt = $db->prepare("INSERT OR REPLACE INTO ".$class->table." (FULLPKGPATH, VALUE) VALUES (?, ?)");
	$stmt->execute($pkgpath, $value);
}

# Generic handling for any blank-separated list
package ListVar;
our @ISA=(qw(SecondaryVar));

sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	AnyVar->add($pkgpath, $db, $var, $value, $rowid);
	for my $d (main::words $value) {
		$class->addvalue($pkgpath, $db, $d) if $d ne '';
	}
}

package QuotedListVar;
our @ISA=(qw(SecondaryVar));

sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	AnyVar->add($pkgpath, $db, $var, $value, $rowid);
	my @l = (main::words $value);
	while (my $v = shift @l) {
		while ($v =~ m/^[^']*\'[^']*$/ || $v =~m/^[^"]*\"[^"]*$/) {
			$v.=' '.shift @l;
		}
		if ($v =~ m/^\"(.*)\"$/) {
		    $v = $1;
		}
		if ($v =~ m/^\'(.*)\'$/) {
		    $v = $1;
		}
		$class->addvalue($pkgpath, $db, $v) if $v ne '';
	}
}

package DefinedListVar;
our @ISA=(qw(ListVar));
sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	return if $value eq '';
	$class->SUPER::add($pkgpath, $db, $var, $value, $rowid);
}

package FlavorsVar;
our @ISA=(qw(DefinedListVar));
sub table() { 'Flavors' }

package CategoriesVar;
our @ISA=(qw(ListVar));
sub table() { 'Categories' }

package MultiVar;
our @ISA=(qw(ListVar));
sub table() { 'Multi' }

package ModulesVar;
our @ISA=(qw(ListVar));
sub table() { 'Modules' }

package ConfigureVar;
our @ISA=(qw(DefinedListVar));
sub table() { 'Configure' }

package ConfigureArgsVar;
our @ISA=(qw(QuotedListVar));
sub table() { 'ConfigureArgs' }

package WantlibVar;
our @ISA=(qw(ListVar));
sub table() { 'Wantlib' }
sub addvalue
{
	my ($class, $pkgpath, $db, $value) = @_;
	$class->SUPER::addvalue($pkgpath, $db, $value);
	if ($value =~ m/\.(?:\>?\=)?\d+\.\d+$/) {
		$class->SUPER::addvalue($pkgpath, $db, $`);
	} elsif ($value =~ m/\.(?:\>?\=)?\d+$/) {
		$class->SUPER::addvalue($pkgpath, $db, $`);
	}
}

package FileVar;
our @ISA=(qw(SecondaryVar));

sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	AnyVar->add($pkgpath, $db, $var, $value, $rowid);
	open my $file, '<', $value or return;
	local $/ = undef;
	$class->addvalue($pkgpath, $db, <$file>);
}

sub table() { 'Descr' }

package SharedLibsVar;
our @ISA=(qw(AnyVar));

sub add
{
	my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
	$class->SUPER::add($pkgpath, $db, $var, $value, $rowid);
	my $stmt = $db->prepare("INSERT INTO Shared_Libs (FULLPKGPATH, LIBNAME, VERSION) VALUES (?, ?, ?)");
	my %t = main::words($value);
	while (my ($k, $v) = each %t) {
		$stmt->execute($pkgpath, $k, $v);
	}
}


package main;

use DBI;

my $dbname;
if (@ARGV > 0) {
	$dbname = shift;
} else {
	$dbname = 'sqlports';
}

my $db =DBI->connect("dbi:SQLite:dbname=$dbname", '', '', {AutoCommit => 0});
my $vars = {
    AUTOCONF_VERSION => 'AnyVar',
    AUTOMAKE_VERSION => 'AnyVar',
    BROKEN => 'AnyVar',
    BUILD_DEPENDS => 'BuildDependsVar',
    CATEGORIES=> 'CategoriesVar',
    COMMENT => 'AnyVar',
    CONFIGURE_ARGS => 'ConfigureArgsVar',
    CONFIGURE_STYLE => 'ConfigureVar',
    DESCR => 'FileVar',
    PACKAGING => 'AnyVar',
    DISTFILES => 'AnyVar',
    DISTNAME => 'AnyVar',
    DIST_SUBDIR => 'DefinedVar',
    FLAVORS => 'FlavorsVar',
    FULLPKGNAME => 'AnyVar',
    HOMEPAGE => 'AnyVar',
    IS_INTERACTIVE => 'AnyVar',
    LIB_DEPENDS => 'LibDependsVar',
    MAINTAINER=> 'AnyVar',
    MASTER_SITES => 'AnyVar',
    MASTER_SITES0 => 'AnyVar',
    MASTER_SITES1 => 'AnyVar',
    MASTER_SITES2 => 'AnyVar',
    MASTER_SITES3 => 'AnyVar',
    MASTER_SITES4=> 'AnyVar',
    MASTER_SITES5 => 'AnyVar',
    MASTER_SITES6 => 'AnyVar',
    MASTER_SITES7 => 'AnyVar',
    MASTER_SITES8 => 'AnyVar',
    MASTER_SITES9=> 'AnyVar',
    MODULES => 'ModulesVar',
    MULTI_PACKAGES => 'MultiVar',
    NO_BUILD => 'YesNoVar',
    NO_REGRESS => 'YesNoVar',
    ONLY_FOR_ARCHS => 'AnyVar',
    PACKAGES => 'AnyVar',
    PERMIT_DISTFILES_CDROM => 'AnyVar',
    PERMIT_DISTFILES_FTP=> 'AnyVar',
    PERMIT_PACKAGE_CDROM => 'AnyVar',
    PERMIT_PACKAGE_FTP=> 'AnyVar',
    PKGNAME => 'AnyVar',
    PKG_ARCH => 'AnyVar',
    PSEUDO_FLAVORS => 'DefinedVar',
    REGRESS_DEPENDS => 'RegressDependsVar',
    REGRESS_IS_INTERACTIVE => 'AnyVar',
    RUN_DEPENDS => 'RunDependsVar',
    SEPARATE_BUILD => 'AnyVar',
    SHARED_LIBS => 'SharedLibsVar',
    SHARED_ONLY => 'YesNoVar',
    SUBPACKAGE => 'DefinedVar',
    SUPDISTFILES => 'AnyVar',
    USE_GMAKE => 'YesNoVar',
    USE_LIBTOOL => 'YesNoVar',
    USE_MOTIF => 'YesNoVar',
    USE_X11 => 'YesNoVar',
    WANTLIB => 'WantlibVar',
};

# create the various tables, dropping old versions

for my $t (qw(Categories Flavors Multi Modules Configure ConfigureArgs Wantlib)) {
    $db->do("DROP TABLE IF EXISTS $t");
    $db->do("CREATE TABLE $t (FULLPKGPATH TEXT NOT NULL, VALUE TEXT NOT NULL, UNIQUE(FULLPKGPATH, VALUE))");
}
$db->do("DROP TABLE IF EXISTS Depends");
$db->do("CREATE TABLE Depends (FULLPKGPATH TEXT NOT NULL, FULLDEPENDS TEXT NOT NULL, DEPENDSPATH TEXT NOT NULL, TYPE TEXT NOT NULL)");
$db->do("DROP TABLE IF EXISTS Shared_Libs");
$db->do("CREATE TABLE Shared_Libs (FULLPKGPATH TEXT NOT NULL, LIBNAME TEXT NOT NULL, VERSION TEXT NOT NULL, UNIQUE (FULLPKGPATH, LIBNAME))");
$db->do("DROP TABLE IF EXISTS Ports");
$db->do("CREATE TABLE Ports (FULLPKGPATH TEXT NOT NULL PRIMARY KEY, ".
	join(',', (map {$_." TEXT"} (keys %$vars))).")");
$db->do("DROP TABLE IF EXISTS Paths");
$db->do("CREATE TABLE Paths (FULLPKGPATH TEXT NOT NULL PRIMARY KEY, PKGPATH TEXT NOT NULL)");
$db->do("DROP TABLE IF EXISTS Descr");
$db->do("CREATE TABLE Descr (FULLPKGPATH TEXT NOT NULL PRIMARY KEY, VALUE TEXT NOT NULL)");
$db->commit();

my $stmt = $db->prepare("SELECT RowID FROM Ports WHERE FULLPKGPATH=(?)");
my $stmt2= $db->prepare("INSERT INTO Ports (FULLPKGPATH) VALUES (?)");
my $stmt3= $db->prepare("INSERT INTO Paths (FULLPKGPATH, PKGPATH) VALUES (?, ?)");

my $i = 0;
my $rowid;
my $lastkey;
while (<STDIN>) {
	chomp;
	# kill noise
	if (m/^\=\=\=/) {
		print $_, "\n";
		next;
	}
	next unless m/^(.*?)\.([A-Z][A-Z_0-9]*)\=(.*)$/;

	my ($key, $var, $value) = ($1, $2, $3);
	# strip extra quotes
	if ($value =~ m/^\"(.*)\"$/) {
		$value = $1;
	}

	if (!(defined $lastkey) || $key ne $lastkey) {
		# get rowid for existing value
		$stmt->execute($key);
		my $z = $stmt->fetchall_arrayref;
		if (@$z == 0) {
			# if none, we create one
			$stmt2->execute($key);
			my $path = $key;
			$path =~ s/\,.*//;
			$stmt3->execute($key, $path);
			$stmt->execute($key);
			$z = $stmt->fetchall_arrayref;
		}
		$rowid = $z->[0]->[0];
		$lastkey = $key;
	}
	$vars->{$var}->add($key, $db, $var, $value, $rowid);

	# and we commit just once every 1000 transactions, for efficiency
	$i++;
	if ($i % 1000 == 0) {
		$db->commit();
	}
}
$db->commit();