Example script to convert values from the ports tree into an sqlite database.
Discussed with robi/bernd/steven.
This commit is contained in:
espie
parent
24960db701
commit
1aecbb09e3
282
infrastructure/package/mksqlitedb
Normal file
282
infrastructure/package/mksqlitedb
Normal file
@ -0,0 +1,282 @@
|
||||
#! /usr/bin/perl
|
||||
# $OpenBSD: mksqlitedb,v 1.1 2006/07/09 11:00:53 espie Exp $
|
||||
#
|
||||
# Copyright (c) 2006 Marc Espie <espie@openbsd.org>
|
||||
#
|
||||
# Permission to use, copy, modify, and distribute this software for any
|
||||
# purpose with or without fee is hereby granted, provided that the above
|
||||
# copyright notice and this permission notice appear in all copies.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
# example script that shows how to store all variable values into a
|
||||
# database, using SQLite for that purpose.
|
||||
#
|
||||
# usage: cd /usr/ports && make dump-all-vars |mksqlitedb
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
|
||||
# use a Template Method approach to store the variable values.
|
||||
|
||||
# rule: we store each value in the main table, after converting YesNo
|
||||
# variables to undef/1. Then, in addition, we process specific variables
|
||||
# to store them in secundary tables (because of one/many associations).
|
||||
|
||||
package AnyVar;
|
||||
|
||||
sub add
|
||||
{
|
||||
my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
|
||||
my $stmt=$db->prepare("UPDATE Ports SET $var=(?) WHERE RowID=(?)");
|
||||
$stmt->execute($value, $rowid);
|
||||
}
|
||||
|
||||
package YesNoVar;
|
||||
our @ISA=(qw(AnyVar));
|
||||
|
||||
sub add
|
||||
{
|
||||
my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
|
||||
$class->SUPER::add($pkgpath, $db, $var, $value =~ m/^Yes/i ? 1 : undef, $rowid);
|
||||
}
|
||||
|
||||
|
||||
# all the dependencies are converted into list. Stuff like LIB_DEPENDS will
|
||||
# end up being treated as WANTLIB as well.
|
||||
|
||||
package DependsVar;
|
||||
sub add
|
||||
{
|
||||
my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
|
||||
AnyVar->add($pkgpath, $db, $var, $value, $rowid);
|
||||
for my $depends (split /\s+/, $value) {
|
||||
my ($libs, $pkgspec, $pkgpath2, $rest) = split(/\:/, $depends);
|
||||
my $stmt = $db->prepare("INSERT INTO Depends (PKGPATH, FULLDEPENDS, DEPENDSPATH, TYPE) VALUES (?, ?, ?, ?)");
|
||||
$stmt->execute($pkgpath, $depends, $pkgpath2, $class->type());
|
||||
for my $lib (split(/,/, $libs)) {
|
||||
$class->addlib($pkgpath, $db, $lib);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub addlib
|
||||
{
|
||||
}
|
||||
|
||||
package LibDependsVar;
|
||||
our @ISA=(qw(DependsVar));
|
||||
sub type() { 'L' }
|
||||
|
||||
sub addlib
|
||||
{
|
||||
my ($class, $pkgpath, $db, $lib) = @_;
|
||||
WantlibVar->addvalue($pkgpath, $db, $lib);
|
||||
}
|
||||
|
||||
package RunDependsVar;
|
||||
our @ISA=(qw(DependsVar));
|
||||
sub type() { 'R' }
|
||||
|
||||
package BuildDependsVar;
|
||||
our @ISA=(qw(DependsVar));
|
||||
sub type() { 'B' }
|
||||
|
||||
package RegressDependsVar;
|
||||
our @ISA=(qw(DependsVar));
|
||||
sub type() { 'Regress' }
|
||||
|
||||
# Stuff that gets stored in another table as well
|
||||
package SecundaryVar;
|
||||
sub addvalue
|
||||
{
|
||||
my ($class, $pkgpath, $db, $value) = @_;
|
||||
my $stmt = $db->prepare("INSERT OR REPLACE INTO ".$class->table."(PKGPATH, VALUE) VALUES (?, ?)");
|
||||
$stmt->execute($pkgpath, $value);
|
||||
}
|
||||
|
||||
# Generic handling for any blank-separated list
|
||||
package ListVar;
|
||||
our @ISA=(qw(SecundaryVar));
|
||||
|
||||
sub add
|
||||
{
|
||||
my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
|
||||
AnyVar->add($pkgpath, $db, $var, $value, $rowid);
|
||||
for my $d (split /\s+/, $value) {
|
||||
$class->addvalue($pkgpath, $db, $d);
|
||||
}
|
||||
}
|
||||
|
||||
package FlavorsVar;
|
||||
our @ISA=(qw(ListVar));
|
||||
sub table() { 'Flavors' }
|
||||
|
||||
package CategoriesVar;
|
||||
our @ISA=(qw(ListVar));
|
||||
sub table() { 'Categories' }
|
||||
|
||||
package MultiVar;
|
||||
our @ISA=(qw(ListVar));
|
||||
sub table() { 'Multi' }
|
||||
|
||||
package ModulesVar;
|
||||
our @ISA=(qw(ListVar));
|
||||
sub table() { 'Modules' }
|
||||
|
||||
package ConfigureVar;
|
||||
our @ISA=(qw(ListVar));
|
||||
sub table() { 'Configure' }
|
||||
|
||||
package WantlibVar;
|
||||
our @ISA=(qw(ListVar));
|
||||
sub table() { 'Wantlib' }
|
||||
sub addvalue
|
||||
{
|
||||
my ($class, $pkgpath, $db, $value) = @_;
|
||||
$class->SUPER::addvalue($pkgpath, $db, $value);
|
||||
if ($value =~ m/\.\d+\.\d+$/) {
|
||||
$class->SUPER::addvalue($pkgpath, $db, $`);
|
||||
} elsif ($value =~ m/\.\d+$/) {
|
||||
$class->SUPER::addvalue($pkgpath, $db, $`);
|
||||
}
|
||||
}
|
||||
|
||||
package FileVar;
|
||||
our @ISA=(qw(SecundaryVar));
|
||||
|
||||
sub add
|
||||
{
|
||||
my ($class, $pkgpath, $db, $var, $value, $rowid) = @_;
|
||||
AnyVar->add($pkgpath, $db, $var, $value, $rowid);
|
||||
open my $file, '<', $value or return;
|
||||
local $/ = undef;
|
||||
$class->addvalue($pkgpath, $db, <$file>);
|
||||
}
|
||||
|
||||
sub table() { 'Descr' }
|
||||
|
||||
package main;
|
||||
|
||||
use DBI;
|
||||
|
||||
my $dbname;
|
||||
if (@ARGV > 0) {
|
||||
$dbname = shift;
|
||||
} else {
|
||||
$dbname = 'sqlports';
|
||||
}
|
||||
|
||||
my $db =DBI->connect("dbi:SQLite:dbname=$dbname", '', '', {AutoCommit => 0});
|
||||
my $vars = {
|
||||
BROKEN => 'AnyVar',
|
||||
BUILD_DEPENDS => 'BuildDependsVar',
|
||||
CATEGORIES=> 'CategoriesVar',
|
||||
COMMENT => 'AnyVar',
|
||||
CONFIGURE_STYLE => 'ConfigureVar',
|
||||
DESCR => 'FileVar',
|
||||
PACKAGING => 'AnyVar',
|
||||
DISTFILES => 'AnyVar',
|
||||
DISTNAME => 'AnyVar',
|
||||
DIST_SUBDIR => 'AnyVar',
|
||||
FLAVORS => 'FlavorsVar',
|
||||
HOMEPAGE => 'AnyVar',
|
||||
IS_INTERACTIVE => 'AnyVar',
|
||||
LIB_DEPENDS => 'LibDependsVar',
|
||||
MAINTAINER=> 'AnyVar',
|
||||
MASTER_SITES => 'AnyVar',
|
||||
MASTER_SITES0 => 'AnyVar',
|
||||
MASTER_SITES1 => 'AnyVar',
|
||||
MASTER_SITES2 => 'AnyVar',
|
||||
MASTER_SITES3 => 'AnyVar',
|
||||
MASTER_SITES4=> 'AnyVar',
|
||||
MASTER_SITES5 => 'AnyVar',
|
||||
MASTER_SITES6 => 'AnyVar',
|
||||
MASTER_SITES7 => 'AnyVar',
|
||||
MASTER_SITES8 => 'AnyVar',
|
||||
MASTER_SITES9=> 'AnyVar',
|
||||
MODULES => 'ModulesVar',
|
||||
MULTI_PACKAGES => 'MultiVar',
|
||||
NO_BUILD => 'YesNoVar',
|
||||
NO_REGRESS => 'YesNoVar',
|
||||
ONLY_FOR_ARCHS => 'AnyVar',
|
||||
PACKAGES => 'AnyVar',
|
||||
PERMIT_DISTFILES_CDROM => 'AnyVar',
|
||||
PERMIT_DISTFILES_FTP=> 'AnyVar',
|
||||
PERMIT_PACKAGE_CDROM => 'AnyVar',
|
||||
PERMIT_PACKAGE_FTP=> 'AnyVar',
|
||||
PKGNAME => 'AnyVar',
|
||||
PKG_ARCH => 'AnyVar',
|
||||
PSEUDO_FLAVORS => 'AnyVar',
|
||||
REGRESS_DEPENDS => 'RegressDependsVar',
|
||||
REGRESS_IS_INTERACTIVE => 'AnyVar',
|
||||
RUN_DEPENDS => 'RunDependsVar',
|
||||
SEPARATE_BUILD => 'AnyVar',
|
||||
SHARED_LIBS => 'AnyVar',
|
||||
SHARED_ONLY => 'YesNoVar',
|
||||
SUBPACKAGE => 'AnyVar',
|
||||
USE_GMAKE => 'YesNoVar',
|
||||
USE_LIBTOOL => 'YesNoVar',
|
||||
USE_MOTIF => 'YesNoVar',
|
||||
WANTLIB => 'WantlibVar',
|
||||
};
|
||||
|
||||
# create the various tables, dropping old versions
|
||||
|
||||
for my $t (qw(Categories Flavors Multi Modules Configure Wantlib Descr)) {
|
||||
$db->do("DROP TABLE $t");
|
||||
$db->do("CREATE TABLE $t (PKGPATH TEXT NOT NULL, VALUE TEXT NOT NULL, UNIQUE(PKGPATH, VALUE))");
|
||||
}
|
||||
$db->do("DROP TABLE Depends");
|
||||
$db->do("CREATE TABLE Depends (PKGPATH TEXT NOT NULL, FULLDEPENDS TEXT NOT NULL, DEPENDSPATH TEXT NOT NULL, TYPE TEXT NOT NULL)");
|
||||
$db->do("DROP TABLE Ports");
|
||||
$db->do("CREATE TABLE Ports (PKGPATH TEXT PRIMARY KEY, ".
|
||||
join(',', (map {$_." TEXT"} (keys %$vars))).")");
|
||||
$db->commit();
|
||||
|
||||
my $stmt = $db->prepare("SELECT RowID FROM Ports WHERE PKGPATH=(?)");
|
||||
my $stmt2= $db->prepare("INSERT INTO Ports (PKGPATH) VALUES (?)");
|
||||
|
||||
my $i = 0;
|
||||
while (<STDIN>) {
|
||||
chomp;
|
||||
# kill noise
|
||||
if (m/^\=\=\=/) {
|
||||
print $_, "\n";
|
||||
next;
|
||||
}
|
||||
next unless m/^(.*?)\.([A-Z][A-Z_0-9]*)\=(.*)$/;
|
||||
|
||||
my ($key, $var, $value) = ($1, $2, $3);
|
||||
# strip extra quotes
|
||||
if ($value =~ m/^\"(.*)\"/) {
|
||||
$value = $1;
|
||||
}
|
||||
|
||||
# get rowid for existing value
|
||||
$stmt->execute($key);
|
||||
my $z = $stmt->fetchall_arrayref;
|
||||
if (@$z == 0) {
|
||||
# if none, we create one
|
||||
$stmt2->execute($key);
|
||||
$stmt->execute($key);
|
||||
$z = $stmt->fetchall_arrayref;
|
||||
}
|
||||
my $rowid = $z->[0]->[0];
|
||||
$vars->{$var}->add($key, $db, $var, $value, $rowid);
|
||||
|
||||
# and we commit just once every 1000 transactions, for efficiency
|
||||
$i++;
|
||||
if ($i % 1000 == 0) {
|
||||
$db->commit();
|
||||
}
|
||||
}
|
||||
$db->commit();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user