From 1aecbb09e365ecd9f15ff459beeff9920b9cea3b Mon Sep 17 00:00:00 2001 From: espie Date: Sun, 9 Jul 2006 11:00:53 +0000 Subject: [PATCH] Example script to convert values from the ports tree into an sqlite database. Discussed with robi/bernd/steven. --- infrastructure/package/mksqlitedb | 282 ++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 infrastructure/package/mksqlitedb diff --git a/infrastructure/package/mksqlitedb b/infrastructure/package/mksqlitedb new file mode 100644 index 00000000000..318cf0306d7 --- /dev/null +++ b/infrastructure/package/mksqlitedb @@ -0,0 +1,282 @@ +#! /usr/bin/perl +# $OpenBSD: mksqlitedb,v 1.1 2006/07/09 11:00:53 espie Exp $ +# +# Copyright (c) 2006 Marc Espie +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# example script that shows how to store all variable values into a +# database, using SQLite for that purpose. +# +# usage: cd /usr/ports && make dump-all-vars |mksqlitedb + +use strict; +use warnings; + + +# use a Template Method approach to store the variable values. + +# rule: we store each value in the main table, after converting YesNo +# variables to undef/1. Then, in addition, we process specific variables +# to store them in secundary tables (because of one/many associations). + +package AnyVar; + +sub add +{ + my ($class, $pkgpath, $db, $var, $value, $rowid) = @_; + my $stmt=$db->prepare("UPDATE Ports SET $var=(?) WHERE RowID=(?)"); + $stmt->execute($value, $rowid); +} + +package YesNoVar; +our @ISA=(qw(AnyVar)); + +sub add +{ + my ($class, $pkgpath, $db, $var, $value, $rowid) = @_; + $class->SUPER::add($pkgpath, $db, $var, $value =~ m/^Yes/i ? 1 : undef, $rowid); +} + + +# all the dependencies are converted into list. Stuff like LIB_DEPENDS will +# end up being treated as WANTLIB as well. + +package DependsVar; +sub add +{ + my ($class, $pkgpath, $db, $var, $value, $rowid) = @_; + AnyVar->add($pkgpath, $db, $var, $value, $rowid); + for my $depends (split /\s+/, $value) { + my ($libs, $pkgspec, $pkgpath2, $rest) = split(/\:/, $depends); + my $stmt = $db->prepare("INSERT INTO Depends (PKGPATH, FULLDEPENDS, DEPENDSPATH, TYPE) VALUES (?, ?, ?, ?)"); + $stmt->execute($pkgpath, $depends, $pkgpath2, $class->type()); + for my $lib (split(/,/, $libs)) { + $class->addlib($pkgpath, $db, $lib); + } + } +} + +sub addlib +{ +} + +package LibDependsVar; +our @ISA=(qw(DependsVar)); +sub type() { 'L' } + +sub addlib +{ + my ($class, $pkgpath, $db, $lib) = @_; + WantlibVar->addvalue($pkgpath, $db, $lib); +} + +package RunDependsVar; +our @ISA=(qw(DependsVar)); +sub type() { 'R' } + +package BuildDependsVar; +our @ISA=(qw(DependsVar)); +sub type() { 'B' } + +package RegressDependsVar; +our @ISA=(qw(DependsVar)); +sub type() { 'Regress' } + +# Stuff that gets stored in another table as well +package SecundaryVar; +sub addvalue +{ + my ($class, $pkgpath, $db, $value) = @_; + my $stmt = $db->prepare("INSERT OR REPLACE INTO ".$class->table."(PKGPATH, VALUE) VALUES (?, ?)"); + $stmt->execute($pkgpath, $value); +} + +# Generic handling for any blank-separated list +package ListVar; +our @ISA=(qw(SecundaryVar)); + +sub add +{ + my ($class, $pkgpath, $db, $var, $value, $rowid) = @_; + AnyVar->add($pkgpath, $db, $var, $value, $rowid); + for my $d (split /\s+/, $value) { + $class->addvalue($pkgpath, $db, $d); + } +} + +package FlavorsVar; +our @ISA=(qw(ListVar)); +sub table() { 'Flavors' } + +package CategoriesVar; +our @ISA=(qw(ListVar)); +sub table() { 'Categories' } + +package MultiVar; +our @ISA=(qw(ListVar)); +sub table() { 'Multi' } + +package ModulesVar; +our @ISA=(qw(ListVar)); +sub table() { 'Modules' } + +package ConfigureVar; +our @ISA=(qw(ListVar)); +sub table() { 'Configure' } + +package WantlibVar; +our @ISA=(qw(ListVar)); +sub table() { 'Wantlib' } +sub addvalue +{ + my ($class, $pkgpath, $db, $value) = @_; + $class->SUPER::addvalue($pkgpath, $db, $value); + if ($value =~ m/\.\d+\.\d+$/) { + $class->SUPER::addvalue($pkgpath, $db, $`); + } elsif ($value =~ m/\.\d+$/) { + $class->SUPER::addvalue($pkgpath, $db, $`); + } +} + +package FileVar; +our @ISA=(qw(SecundaryVar)); + +sub add +{ + my ($class, $pkgpath, $db, $var, $value, $rowid) = @_; + AnyVar->add($pkgpath, $db, $var, $value, $rowid); + open my $file, '<', $value or return; + local $/ = undef; + $class->addvalue($pkgpath, $db, <$file>); +} + +sub table() { 'Descr' } + +package main; + +use DBI; + +my $dbname; +if (@ARGV > 0) { + $dbname = shift; +} else { + $dbname = 'sqlports'; +} + +my $db =DBI->connect("dbi:SQLite:dbname=$dbname", '', '', {AutoCommit => 0}); +my $vars = { + BROKEN => 'AnyVar', + BUILD_DEPENDS => 'BuildDependsVar', + CATEGORIES=> 'CategoriesVar', + COMMENT => 'AnyVar', + CONFIGURE_STYLE => 'ConfigureVar', + DESCR => 'FileVar', + PACKAGING => 'AnyVar', + DISTFILES => 'AnyVar', + DISTNAME => 'AnyVar', + DIST_SUBDIR => 'AnyVar', + FLAVORS => 'FlavorsVar', + HOMEPAGE => 'AnyVar', + IS_INTERACTIVE => 'AnyVar', + LIB_DEPENDS => 'LibDependsVar', + MAINTAINER=> 'AnyVar', + MASTER_SITES => 'AnyVar', + MASTER_SITES0 => 'AnyVar', + MASTER_SITES1 => 'AnyVar', + MASTER_SITES2 => 'AnyVar', + MASTER_SITES3 => 'AnyVar', + MASTER_SITES4=> 'AnyVar', + MASTER_SITES5 => 'AnyVar', + MASTER_SITES6 => 'AnyVar', + MASTER_SITES7 => 'AnyVar', + MASTER_SITES8 => 'AnyVar', + MASTER_SITES9=> 'AnyVar', + MODULES => 'ModulesVar', + MULTI_PACKAGES => 'MultiVar', + NO_BUILD => 'YesNoVar', + NO_REGRESS => 'YesNoVar', + ONLY_FOR_ARCHS => 'AnyVar', + PACKAGES => 'AnyVar', + PERMIT_DISTFILES_CDROM => 'AnyVar', + PERMIT_DISTFILES_FTP=> 'AnyVar', + PERMIT_PACKAGE_CDROM => 'AnyVar', + PERMIT_PACKAGE_FTP=> 'AnyVar', + PKGNAME => 'AnyVar', + PKG_ARCH => 'AnyVar', + PSEUDO_FLAVORS => 'AnyVar', + REGRESS_DEPENDS => 'RegressDependsVar', + REGRESS_IS_INTERACTIVE => 'AnyVar', + RUN_DEPENDS => 'RunDependsVar', + SEPARATE_BUILD => 'AnyVar', + SHARED_LIBS => 'AnyVar', + SHARED_ONLY => 'YesNoVar', + SUBPACKAGE => 'AnyVar', + USE_GMAKE => 'YesNoVar', + USE_LIBTOOL => 'YesNoVar', + USE_MOTIF => 'YesNoVar', + WANTLIB => 'WantlibVar', +}; + +# create the various tables, dropping old versions + +for my $t (qw(Categories Flavors Multi Modules Configure Wantlib Descr)) { + $db->do("DROP TABLE $t"); + $db->do("CREATE TABLE $t (PKGPATH TEXT NOT NULL, VALUE TEXT NOT NULL, UNIQUE(PKGPATH, VALUE))"); +} +$db->do("DROP TABLE Depends"); +$db->do("CREATE TABLE Depends (PKGPATH TEXT NOT NULL, FULLDEPENDS TEXT NOT NULL, DEPENDSPATH TEXT NOT NULL, TYPE TEXT NOT NULL)"); +$db->do("DROP TABLE Ports"); +$db->do("CREATE TABLE Ports (PKGPATH TEXT PRIMARY KEY, ". +join(',', (map {$_." TEXT"} (keys %$vars))).")"); +$db->commit(); + +my $stmt = $db->prepare("SELECT RowID FROM Ports WHERE PKGPATH=(?)"); +my $stmt2= $db->prepare("INSERT INTO Ports (PKGPATH) VALUES (?)"); + +my $i = 0; +while () { + chomp; + # kill noise + if (m/^\=\=\=/) { + print $_, "\n"; + next; + } + next unless m/^(.*?)\.([A-Z][A-Z_0-9]*)\=(.*)$/; + + my ($key, $var, $value) = ($1, $2, $3); + # strip extra quotes + if ($value =~ m/^\"(.*)\"/) { + $value = $1; + } + + # get rowid for existing value + $stmt->execute($key); + my $z = $stmt->fetchall_arrayref; + if (@$z == 0) { + # if none, we create one + $stmt2->execute($key); + $stmt->execute($key); + $z = $stmt->fetchall_arrayref; + } + my $rowid = $z->[0]->[0]; + $vars->{$var}->add($key, $db, $var, $value, $rowid); + + # and we commit just once every 1000 transactions, for efficiency + $i++; + if ($i % 1000 == 0) { + $db->commit(); + } +} +$db->commit(); +