Make zimHttpServer32 compatible with AVM Fritz!Box routers (freetz)

README.avm.perl

@@ -0,0 +1,38 @@
+As an alternative to building perl in-tree, (see
+https://github.com/dirk-dhu/freetz/tree/master/make/perl
+), here are the steps to do this manually, if your
+freetz distribution lacks make/perl package or have
+trouble using the automated build.
+
+To cross-compile perl and integrate it manually
+into a freetz build, here is a rough guideline,
+tried and tested:
+
+  perl-5.26.2.tar.gz
+  perl-cross-1.1.9.tar.gz   (unpack over perl-5.26.2 source)
+
+compile freetz with the following line in .config to setup all the links for perl
+---------------------------------------------------------------------------------
+  EXTERNAL_OWN_FILES="/usr/bin/corelist /usr/bin/cpan /usr/bin/enc2xs /usr/bin/encguess /usr/bin/h2ph /usr/bin/h2xs /usr/bin/instmodsh /usr/bin/json_pp /usr/bin/libnetcfg /usr/bin/perl /usr/bin/perlbug /usr/bin/perldoc /usr/bin/perlivp /usr/bin/perlthanks /usr/bin/piconv /usr/bin/pl2pm /usr/bin/pod2html /usr/bin/pod2man /usr/bin/pod2text /usr/bin/pod2usage /usr/bin/podchecker /usr/bin/podselect /usr/bin/prove /usr/bin/ptar /usr/bin/ptardiff /usr/bin/ptargrep /usr/bin/shasum /usr/bin/splain /usr/bin/xsubpp /usr/bin/zipdetails /usr/lib/perl5"
+
+compile perl and integrate into freetz image using
+---------------------------------------------
+  cd perl-src-dir
+
+  export PATH="$PATH:$FREETZDIR/toolchain/build/mips_gcc-4.8.5_uClibc-0.9.33.2-nptl/mips-linux-uclibc/bin/"
+
+  ./configure --target=mips-linux-uclibc --target-tools-prefix=mips-linux-uclibc
+  make
+  make DESTDIR=$FREETZDIR/build/modified/external
+
+  # do not distribute man page to embedded system
+  rm -rvf $FREETZDIR/build/modified/external/usr/share/man
+
+  # repack image to reflect modified/external state
+  FREETZ_FWMOD_SKIP_UNPACK=y FREETZ_FWMOD_SKIP_MODIFY=y make firmware-nocompile
+
+
+The resulting firmware will work to run zimHttpServer32 on Fritz!Box
+router hardware in combination with the machine-endianess-tolerant
+zimHttpServer.pl in this repo-sitory.
+

README.avm.xzutils

@@ -0,0 +1,10 @@
+    - Busybox (tested 1.24 and current 1.28) xz applet appears to not
+      work (reports "corrupted data" to the terminal) when applied to
+      the xz data extracted from zim files, _but_ the error message
+      is wrong - if enough RAM (physical and swap) is available the
+      xz applet will work just as well as the binary from xz-utils.
+
+    - Thus minimum RAM requirement is 128 MB (physical, and another
+      64 to 128 MB using swap) to serve heavily compressed zim files
+      from AVM Fritz!Box routers, i.e. it may not work with 7270 and
+      older devices

README.md

@@ -0,0 +1,26 @@
+
+This repo is DEPRECATED and here for reference purpose only, it may be deleted from github at any time in the future.
+
+The clone was originally done with the intention
+of integration into freetz, thus the content of
+the few commits in this repo has basically moved
+to
+https://github.com/cm8/freetz/commits/make/zimhttpserver
+
+Due to the way freetz packages work, there is not
+a full clone - make/zimhttpserver/zimhttpserver.mk
+rather refers to the original upstream repo,
+instead of cloning it fully into freetz. The
+reason for freetz to do it this way is, that
+it is unclear if a clone is needed until the
+user has configured freetz build system.
+
+After some grace period, /this/ fork will probably
+be deleted, because the original intention of
+integration into freetz is now finished. If you
+have bookmarked it for some reason, take a look
+at
+https://github.com/cm8/freetz/commit/a89bb115d2d9ec47f7ad85dc3ec24a286b6b1a64
+
+This commit basically squashes the changes done
+to this repo since fork time.

zimHttpServer.pl

@@ -10,18 +10,24 @@
 
 use strict;
 use Socket;
-my %article;
+use List::Util qw( min max );
 
-my $UNAME=`uname -m`;
-chomp($UNAME);
-print "UNAME = [$UNAME]\n";
+my %article;
 my $XZ;
-if (-e "./$UNAME/xz") {
-    $XZ="./$UNAME/xz";
-    print "Detected $UNAME, using local xz binary at $XZ\n";
-} else {
-    print "Using system xz binary\n";
+if (-e "/usr/bin/xz" or -e "/bin/xz") {
     $XZ="xz";
+    print "Using system xz binary.\n";
+} else {
+    my $UNAME=`uname -m`;
+    chomp($UNAME);
+    print "UNAME = [$UNAME]\n";
+
+    if (-e "./$UNAME/xz") {
+        $XZ="./$UNAME/xz";
+        print "Using local xz binary at $XZ\n";
+    } else {
+        die "No suitable xz decompressor found\n";
+    }
 }
 
 # open «file.zim». For more information see internet «openzim.org»
@@ -73,17 +79,17 @@ sub xseek {
 my %header;
 xseek(\*FILE, 0, 0); # no necesary because it must be it
 xread(\*FILE, $_, 4); $header{"magicNumber"} = unpack("c*"); # ZIM\x04
-xread(\*FILE, $_, 4); $header{"version"} = unpack("I");
+xread(\*FILE, $_, 4); $header{"version"} = unpack("V");
 xread(\*FILE, $_, 16); $header{"uuid"} = unpack("H*");
-xread(\*FILE, $_, 4); $header{"articleCount"} = unpack("I");
-xread(\*FILE, $_, 4); $header{"clusterCount"} = unpack("I");
-
-xread(\*FILE, $_, 4); $header{"urlPtrPos"} = unpack("I");     xread(\*FILE, $_, 4); $header{"zero1"} = unpack("I");
-xread(\*FILE, $_, 4); $header{"titlePtrPos"} = unpack("I");   xread(\*FILE, $_, 4); $header{"zero2"} = unpack("I");
-xread(\*FILE, $_, 4); $header{"clusterPtrPos"} = unpack("I"); xread(\*FILE, $_, 4); $header{"zero3"} = unpack("I");
-xread(\*FILE, $_, 4); $header{"mimeListPos"} = unpack("I");   xread(\*FILE, $_, 4); $header{"zero4"} = unpack("I");
-xread(\*FILE, $_, 4); $header{"mainPage"} = unpack("I");
-xread(\*FILE, $_, 4); $header{"layoutPage"} = unpack("I");
+xread(\*FILE, $_, 4); $header{"articleCount"} = unpack("V");
+xread(\*FILE, $_, 4); $header{"clusterCount"} = unpack("V");
+
+xread(\*FILE, $_, 4); $header{"urlPtrPos"} = unpack("V");     xread(\*FILE, $_, 4); $header{"zero1"} = unpack("V");
+xread(\*FILE, $_, 4); $header{"titlePtrPos"} = unpack("V");   xread(\*FILE, $_, 4); $header{"zero2"} = unpack("V");
+xread(\*FILE, $_, 4); $header{"clusterPtrPos"} = unpack("V"); xread(\*FILE, $_, 4); $header{"zero3"} = unpack("V");
+xread(\*FILE, $_, 4); $header{"mimeListPos"} = unpack("V");   xread(\*FILE, $_, 4); $header{"zero4"} = unpack("V");
+xread(\*FILE, $_, 4); $header{"mainPage"} = unpack("V");
+xread(\*FILE, $_, 4); $header{"layoutPage"} = unpack("V");
 xread(\*FILE, $_, 8); $header{"checksumPos"} = unpack("H*");
 
 # Check that none of the *PtrPos values exceed 32-bits
@@ -185,8 +191,8 @@ sub url_pointer{
 	my $pos = $header{"urlPtrPos"};
 	$pos += $article*8;
 	xseek(\*FILE, $pos, 0);
-        xread(\*FILE, $_, 4); my $ret = unpack("I");
-        xread(\*FILE, $_, 4); my $ret64 = unpack("I");
+        xread(\*FILE, $_, 4); my $ret = unpack("V");
+        xread(\*FILE, $_, 4); my $ret64 = unpack("V");
 	return ($ret64, $ret);
 }
 
@@ -200,7 +206,7 @@ sub title_pointer{
 	my $pos = $header{"titlePtrPos"};
 	$pos += $article_by_title*4;
 	xseek(\*FILE, $pos,0);
-	xread(\*FILE, $_, 4); my $ret = unpack("I");
+	xread(\*FILE, $_, 4); my $ret = unpack("V");
 	return $ret;
 }
 
@@ -221,15 +227,17 @@ sub entry{
         }
 	xseek(\*FILE, $pos,1);
 
-	xread(\*FILE, $_, 2); $article{"mimetype"} = unpack("s");
+	# use this to read the field with perl versions < 5.10
+	# xread(\*FILE, $_, 2); $article{"mimetype"} = unpack("s", pack("S", unpack("v")));
+	xread(\*FILE, $_, 2); $article{"mimetype"} = unpack("s<");
 	xread(\*FILE, $_, 1); $article{"parameter_len"} = unpack("H*");
 	xread(\*FILE, $_, 1); $article{"namespace"} = unpack("a");
-	xread(\*FILE, $_, 4); $article{"revision"} = unpack("I");
+	xread(\*FILE, $_, 4); $article{"revision"} = unpack("V");
 	if($article{"mimetype"} <0){
-		xread(\*FILE, $_, 4); $article{"redirect_index"} = unpack("I");
+		xread(\*FILE, $_, 4); $article{"redirect_index"} = unpack("V");
 	}else{
-		xread(\*FILE, $_, 4); $article{"cluster_number"} = unpack("I");
-		xread(\*FILE, $_, 4); $article{"blob_number"} = unpack("I");
+		xread(\*FILE, $_, 4); $article{"cluster_number"} = unpack("V");
+		xread(\*FILE, $_, 4); $article{"blob_number"} = unpack("V");
 	}
         $article{"url"} = get_null_string();
         $article{"title"} = get_null_string();
@@ -253,8 +261,8 @@ sub cluster_pointer{
 	my $pos = $header{"clusterPtrPos"};
 	$pos += $cluster*8;
 	xseek(\*FILE, $pos,0);
-	xread(\*FILE, $_, 4); my $ret = unpack("I");
-        xread(\*FILE, $_, 4); my $ret64 = unpack("I");
+	xread(\*FILE, $_, 4); my $ret = unpack("V");
+        xread(\*FILE, $_, 4); my $ret64 = unpack("V");
 	return ($ret64, $ret);
 }
 
@@ -297,30 +305,49 @@ sub cluster_blob{
 	if($cluster{"compression_type"} == 4){
 		my $data_compressed;
 		xread(\*FILE, $data_compressed, $size);
+
+		# The following line breaks because it includes the absolute path of arg0
+		#my $file = "/tmp/$ARGV[0]_cluster$cluster-pid$$";
 		my $file = "/tmp/$$-cluster-$cluster";
-# The following line breaks because it includes the absolute path of arg0
-#		my $file = "/tmp/$ARGV[0]_cluster$cluster-pid$$";
+
 		open(DATA, ">$file.xz");
 		print DATA $data_compressed;
 		close(DATA);
-		`$XZ -d -f $file.xz`;
-		open(DATA, "$file");
-#	my $blob1;
-#	xread(DATA, $blob1, 4);
-#	my $blob_count = int($blob1/4);
-		seek(DATA, $blob*4, 0);
-		read(DATA, $_, 4); my $posStart = unpack("I");
-		read(DATA, $_, 4); my $posEnd = unpack("I");
-		seek(DATA, $posStart, 0);
-		read(DATA, $ret, $posEnd-$posStart);
+
+		open(DATA, "-|", "$XZ -d -c $file.xz");
+
+		my $px = $blob * 4;
+		my $p = 0;
+		$p += sysread(DATA, $_, min(4*1024, $px-$p)), while $p < $px;
+
+		$px += 4;
+		$p += sysread(DATA, $_, 4);
+		my $posStart = unpack("V");
+
+		$px += 4;
+		$p += sysread(DATA, $_, 4);
+		my $posEnd = unpack("V");
+
+		$px += ($posStart - $px);
+		$p += sysread(DATA, $_, min(4*1024, $px-$p)), while $p < $px;
+
+		$ret = "x" x (124*1024);
+		$ret = "";
+		$px += ($posEnd - $posStart);
+		while( $p < $px ) {
+			$p += sysread(DATA, $_, min(4*1024, $px-$p));
+			$ret .= $_;
+		}
+
 		close(DATA);
-		`rm $file`;
+		`rm $file.xz`;
+
 		return $ret;
         } elsif ($cluster{"compression_type"} == 1) {
 		my $data;
 		xread(\*FILE, $data, $size);
-		$_ = substr $data, $blob*4, 4;my $posStart = unpack("I");
-		$_ = substr $data, $blob*4+4, 4;my $posEnd = unpack("I");
+		$_ = substr $data, $blob*4, 4;my $posStart = unpack("V");
+		$_ = substr $data, $blob*4+4, 4;my $posEnd = unpack("V");
 		$ret = substr $data, $posStart, $posEnd-$posStart;
 		return $ret;
 	} else {