Merge branch 'Odd0002-master'

data/optimize_data.sh

@@ -1,23 +1,74 @@
-#!/bin/sh
+#!/bin/bash
 #
 # (C) 2013 Lauri Kasanen, under the GPLv3
+# (C) 2014-2015 Odd0002, under the GPLv3
 #
 # Script to optimize the data, currently PNG, JPEG, B3DZ.
+# experimental B3D to B3DZ compression added, not enabled by default
 # Run it before making a release, and after adding new data.
-# Takes 5-10min depending on your cpu.
+
+#Takes 30+ minutes, depending on your cpu or compression options.
 
 # Spaces in filenames are supported.
 
+#define number of threads
+threads=$(nproc)
+
 # Start checks
+#if you do not want to use a program, set the variable for it to false here
+jpegtran=true
+advdef=true
+advzip=true
+optipng=true
 
-fail=0
-which awk >/dev/null && which jpegtran >/dev/null && which advdef >/dev/null && \
-which advzip >/dev/null || fail=1
+#WARNING! SETTING TO TRUE MAY POSSIBLY INCREASE LOAD TIMES ON A SLOW CPU (UNTESTED) OR LEAD TO FILE NOT FOUND ERRORS WHEN RUNNING SUPERTUXKART! 
+compress_b3d=false
+
+
+#---------------------------------------------------------
+# Begin main code
+
+#Check for reqired programs; if a program is not available, disable it or quit the program
+
+#TODO: make awk optional
+if [ ! $(which awk) ]
+then
+	echo "Please install awk. It is required by this program. \nQuitting..."
+	exit 1
+fi
+
+#check for jpegtran
+if [ ! $(which jpegtran) ]
+then
+	echo "jpegtran not installed, therefore it will not be used.  It is included in the package \"libjpeg-progs\"."; jpegtran=false
+	sleep 2
+fi
+
+#check for advdef
+if [ ! $(which advdef) ]
+then
+	echo "advdef is not installed, therefore it will not be used.  It is included in the package \"advancecomp\"."; advdef=false
+	sleep 2
+fi
+
+#check for advzip
+if [ ! $(which advzip) ]
+then
+	echo "advzip is not installed, therefore it will not be used.  It is included in the package \"advancecomp\"."; advzip=false
+	sleep 2
+fi
+
+#check for optipng
+if [ ! $(which optipng) ]
+then
+        echo "optipng is not installed, therefore it will not be used."; optipng=false
+	sleep 2
+fi
 
-[ "$fail" -eq 1 ] && echo "Please install awk, advdef and jpegtran" && exit 1
 
 # Defines
 
+#Internal Field Seperator
 IFS="
 "
 
@@ -25,25 +76,111 @@ export LANG=C
 
 # Go
 
+#store disk usage of files beforehand
 BEFORE=`du -sk | awk '{print $1}'`
 
-for png in `find -name "*.png"`; do
-	advdef -z4 "$png"
+
+#functions for xargs multithreading, used instead of GNU parallel for cross-compatibility
+#TODO: let next set of optimization scripts run if one set is stuck on a single file at the end to decrease total runtime
+
+#optimize PNG's
+optimpng () {
+for arg; do
+        optipng -quiet -o3 "$arg"
+#level 3 = 16 trials, which according to http://optipng.sourceforge.net/pngtech/optipng.html (retrieved October 2014) should be satisfactory for all users
+done
+}
+export -f optimpng
+
+
+#compress PNG in-stream data
+comprpng () {
+for arg; do
+        advdef -z4 "$arg"
+done
+}
+export -f comprpng
+
+
+#optimize and recompress jpeg files (losslessly)
+optimjpg () {
+for arg; do
+	jpegtran -optimize -copy none -outfile "$arg".$$ "$arg"
+	mv "$arg".$$ "$arg"
+done
+}
+export -f optimjpg
+
+
+#recompress b3dz files
+recomprb3dz () {
+for arg; do
+	advzip -z4 "$arg"
+done
+}
+export -f recomprb3dz
+#END MULTITHREADING FUNCTIONS
+
+#lossless png image optimization
+if [ "$optipng" = true ]; then
+	find . -path .svn -prune -o -name "*.png" -print0 | xargs -0 -n 1 -P "$threads" bash -c 'optimpng "$@"' -- #multithread the png optimization
+else echo "optipng not installed. Ignoring commands using optipng..."; sleep 1
+fi
+
+
+#in-stream data/png compression
+if [ "$advdef" = true ]; then
+	find . -path .svn -prune -o -name "*.png" -print0 | xargs -0 -n 1 -P "$threads" bash -c 'comprpng "$@"' -- #multithread image compression
+else echo "advdef is not installed. Ignoring commands using advdef..."; sleep 1
+fi
+
+
+#lossless jpeg optimization/recompression
+if [ "$jpegtran" = true ]; then
+	find . -path .svn -prune -o -name "*.jpg" -print0 | xargs -0 -n 1 -P "$threads" bash -c 'optimjpg "$@"' -- #multithread jpg compression and optimization
+else echo "jpegtran not installed. Ignoring commands using jpegtran..."; sleep 1
+fi
+
+
+#b3dz to b3dz compression 
+#WARNING: BETA, MAY CAUSE MISSING FILE WARNINGS!
+if [ "$compress_b3d" = true ]; then
+	for xmls in $(find . -name "*.xml"); do
+		sed 's/b3d/b3dz/g' "$xmls" > "$xmls".$$
+		mv "$xmls".$$ "$xmls"
+		#echo "$xmls"
+		sed 's/b3dzz/b3dz/g' "$xmls" > "$xmls".$$
+		mv "$xmls".$$ "$xmls"
+		sed 's/b3dzz/b3dz/g' "$xmls" > "$xmls".$$
+		mv "$xmls".$$ "$xmls"
 	done
 
-for jpeg in `find -name "*.jpg"`; do
-	jpegtran -optimize -copy none -outfile "$jpeg".$$ "$jpeg"
-	mv "$jpeg".$$ "$jpeg"
+	find . -name "*.b3d" -execdir zip '{}.zip' '{}' \;
+
+	for b3dzip in $(find -name "*.b3d.zip"); do
+		b3dz="${b3dzip%.zip}"
+		#echo "$b3dz"
+		mv "$b3dzip" "${b3dz}z"
 	done
 
-for b3dz in `find -name "*.b3dz"`; do
-	advzip -z4 "$b3dz"
-done
+	find . -type d -name "models" -prune -o -name "*.b3d" -print0 | xargs -0 rm
+
+else echo "b3d to b3dz compression disabled.  Ignoring actions..."; sleep 1
+fi
+
+#b3dz file stream compression
+if [ "$advzip" = true ]; then
+	find . -path .svn -prune -o -name "*.b3dz" -print0 | xargs -0 -n 1 -P "$threads" bash -c 'recomprb3dz "$@"' -- #multithread b3dz recompression
+else echo "advzip not installed. Ignoring commands using advzip..."; sleep 1
+fi
+
 
 # Add optimizations for other types if necessary
 
+# get and store new disk usage info
 AFTER=`du -sk | awk '{print $1}'`
 
+
 # Print stats out
-echo $BEFORE $AFTER | awk '{print "Before: " $1/1024 "mb, after: " $2/1024 "mb" }'
+echo $BEFORE $AFTER | awk '{print "Size before: " $1/1024 "mb;  Size after: " $2/1024 "mb" }'
 echo $BEFORE $AFTER | awk '{print "Saved " (1-($2/$1)) * 100 "%" }'