From 4b01747aa5f29c28d87cab1b8112171d7c9be69c Mon Sep 17 00:00:00 2001 From: obecian Date: Sun, 9 Sep 2001 21:57:12 +0000 Subject: [PATCH] crawl-0.1b import - provos@ ok The crawl utility starts a depth-first traversal of the web at the specified URLs. It stores all JPEG images that match the configured constraints. Crawl is fairly fast and allows for graceful termination. After terminating crawl, it is possible to restart it at exactly the same spot where it was terminated. Crawl keeps a persistent database that allows multiple crawls without revisiting sites. The main reason for writing crawl was the lack of simple open source web crawlers. Crawl is only a few thousand lines of code and fairly easy to debug and customize. Features + Saves encountered JPEG images + Image selection based on regular expressions and size contraints + Resume previous crawl after graceful termination + Persistent database of visited URLs + Very small and efficient code + Supports robots.txt --- net/crawl/Makefile | 26 ++++++++++++++++++++++++++ net/crawl/files/md5 | 3 +++ net/crawl/patches/patch-Makefile_in | 28 ++++++++++++++++++++++++++++ net/crawl/patches/patch-configure | 11 +++++++++++ net/crawl/pkg/DESCR | 21 +++++++++++++++++++++ net/crawl/pkg/PLIST | 3 +++ 6 files changed, 92 insertions(+) create mode 100644 net/crawl/Makefile create mode 100644 net/crawl/files/md5 create mode 100644 net/crawl/patches/patch-Makefile_in create mode 100644 net/crawl/patches/patch-configure create mode 100644 net/crawl/pkg/DESCR create mode 100644 net/crawl/pkg/PLIST diff --git a/net/crawl/Makefile b/net/crawl/Makefile new file mode 100644 index 00000000000..ef2cf3c79ef --- /dev/null +++ b/net/crawl/Makefile @@ -0,0 +1,26 @@ +# $OpenBSD: Makefile,v 1.1.1.1 2001/09/09 21:57:12 obecian Exp $ + +COMMENT= "small and efficient HTTP crawler" + +DISTNAME= crawl-0.1b +CATEGORIES= net +NEED_VERSION= 1.406 + +HOMEPAGE= http://www.monkey.org/~provos/crawl/ + +MAINTAINER= Mark Grimes + +PERMIT_PACKAGE_CDROM= Yes +PERMIT_PACKAGE_FTP= Yes +PERMIT_DISTFILES_CDROM= Yes +PERMIT_DISTFILES_FTP= Yes + +MASTER_SITES= http://www.monkey.org/~provos/ + +BUILD_DEPENDS= ${LOCALBASE}/lib/libevent.a::devel/libevent + +CONFIGURE_STYLE= autoconf + +WRKDIST= ${WRKDIR}/crawl + +.include diff --git a/net/crawl/files/md5 b/net/crawl/files/md5 new file mode 100644 index 00000000000..89741ebd59c --- /dev/null +++ b/net/crawl/files/md5 @@ -0,0 +1,3 @@ +MD5 (crawl-0.1b.tar.gz) = 3809c8b13fb5d629a799a1972185500a +RMD160 (crawl-0.1b.tar.gz) = 6bbf94508728632cd604317e03a25f50aa9930bb +SHA1 (crawl-0.1b.tar.gz) = 033e8dcbe4eddfa2b721c77ca1496261787c4790 diff --git a/net/crawl/patches/patch-Makefile_in b/net/crawl/patches/patch-Makefile_in new file mode 100644 index 00000000000..32252e6a02a --- /dev/null +++ b/net/crawl/patches/patch-Makefile_in @@ -0,0 +1,28 @@ +--- Makefile.in.orig Sun Jul 22 17:24:41 2001 ++++ Makefile.in Sun Jul 22 17:26:44 2001 +@@ -3,12 +3,13 @@ + srcdir = @srcdir@ + VPATH = @srcdir@ + +-install_prefix = + prefix = @prefix@ + exec_prefix = @exec_prefix@ + bindir = @bindir@ + mandir = @mandir@ + ++DESTDIR = ++ + CC = @CC@ + CFLAGS = -Wall @CFLAGS@ @USRINCLUDE@ -I$(srcdir) \ + -I$(srcdir)/missing @DBINC@ @EVENTINC@ +@@ -29,8 +30,8 @@ crawl: $(OBJS) + $(CC) $(CFLAGS) $(INCS) -o $@ $(OBJS) $(LIBS) + + install: all +- $(INSTALL_PROG) -m 755 crawl $(install_prefix)$(bindir) +- $(INSTALL_DATA) crawl.1 $(install_prefix)$(mandir)/man1 ++ $(INSTALL_PROG) -m 755 crawl $(DESTDIR)$(bindir) ++ $(INSTALL_DATA) crawl.1 $(DESTDIR)$(mandir)/man1 + + clean: + rm -f crawl *~ *.core *.db $(OBJS) diff --git a/net/crawl/patches/patch-configure b/net/crawl/patches/patch-configure new file mode 100644 index 00000000000..4b4dc47fce7 --- /dev/null +++ b/net/crawl/patches/patch-configure @@ -0,0 +1,11 @@ +--- configure.orig Thu Jul 19 12:21:50 2001 ++++ configure Thu Jul 19 12:22:01 2001 +@@ -9,7 +9,7 @@ + + # Defaults: + ac_help= +-ac_default_prefix=/usr/local ++ac_default_prefix=${LOCALBASE} + # Any additions from configure.in: + ac_help="$ac_help + --with-libevent=DIR use libevent build directory" diff --git a/net/crawl/pkg/DESCR b/net/crawl/pkg/DESCR new file mode 100644 index 00000000000..916c4f3607c --- /dev/null +++ b/net/crawl/pkg/DESCR @@ -0,0 +1,21 @@ +The crawl utility starts a depth-first traversal of the web at the +specified URLs. It stores all JPEG images that match the configured +constraints. Crawl is fairly fast and allows for graceful termination. +After terminating crawl, it is possible to restart it at exactly the +same spot where it was terminated. Crawl keeps a persistent database +that allows multiple crawls without revisiting sites. + +The main reason for writing crawl was the lack of simple open source +web crawlers. Crawl is only a few thousand lines of code and fairly +easy to debug and customize. + +Features + ++ Saves encountered JPEG images ++ Image selection based on regular expressions and size contraints ++ Resume previous crawl after graceful termination ++ Persistent database of visited URLs ++ Very small and efficient code ++ Supports robots.txt + +WWW: ${HOMEPAGE} diff --git a/net/crawl/pkg/PLIST b/net/crawl/pkg/PLIST new file mode 100644 index 00000000000..4608fc03e35 --- /dev/null +++ b/net/crawl/pkg/PLIST @@ -0,0 +1,3 @@ +@comment $OpenBSD: PLIST,v 1.1.1.1 2001/09/09 21:57:12 obecian Exp $ +bin/crawl +man/man1/crawl.1