diff --git a/README.avm.perl b/README.avm.perl new file mode 100644 index 0000000..8706126 --- /dev/null +++ b/README.avm.perl @@ -0,0 +1,38 @@ +As an alternative to building perl in-tree, (see +https://github.com/dirk-dhu/freetz/tree/master/make/perl +), here are the steps to do this manually, if your +freetz distribution lacks make/perl package or have +trouble using the automated build. + +To cross-compile perl and integrate it manually +into a freetz build, here is a rough guideline, +tried and tested: + + perl-5.26.2.tar.gz + perl-cross-1.1.9.tar.gz (unpack over perl-5.26.2 source) + +compile freetz with the following line in .config to setup all the links for perl +--------------------------------------------------------------------------------- + EXTERNAL_OWN_FILES="/usr/bin/corelist /usr/bin/cpan /usr/bin/enc2xs /usr/bin/encguess /usr/bin/h2ph /usr/bin/h2xs /usr/bin/instmodsh /usr/bin/json_pp /usr/bin/libnetcfg /usr/bin/perl /usr/bin/perlbug /usr/bin/perldoc /usr/bin/perlivp /usr/bin/perlthanks /usr/bin/piconv /usr/bin/pl2pm /usr/bin/pod2html /usr/bin/pod2man /usr/bin/pod2text /usr/bin/pod2usage /usr/bin/podchecker /usr/bin/podselect /usr/bin/prove /usr/bin/ptar /usr/bin/ptardiff /usr/bin/ptargrep /usr/bin/shasum /usr/bin/splain /usr/bin/xsubpp /usr/bin/zipdetails /usr/lib/perl5" + +compile perl and integrate into freetz image using +--------------------------------------------- + cd perl-src-dir + + export PATH="$PATH:$FREETZDIR/toolchain/build/mips_gcc-4.8.5_uClibc-0.9.33.2-nptl/mips-linux-uclibc/bin/" + + ./configure --target=mips-linux-uclibc --target-tools-prefix=mips-linux-uclibc + make + make DESTDIR=$FREETZDIR/build/modified/external + + # do not distribute man page to embedded system + rm -rvf $FREETZDIR/build/modified/external/usr/share/man + + # repack image to reflect modified/external state + FREETZ_FWMOD_SKIP_UNPACK=y FREETZ_FWMOD_SKIP_MODIFY=y make firmware-nocompile + + +The resulting firmware will work to run zimHttpServer32 on Fritz!Box +router hardware in combination with the machine-endianess-tolerant +zimHttpServer.pl in this repo-sitory. + diff --git a/README.avm.xzutils b/README.avm.xzutils new file mode 100644 index 0000000..9257b7f --- /dev/null +++ b/README.avm.xzutils @@ -0,0 +1,10 @@ + - Busybox (tested 1.24 and current 1.28) xz applet appears to not + work (reports "corrupted data" to the terminal) when applied to + the xz data extracted from zim files, _but_ the error message + is wrong - if enough RAM (physical and swap) is available the + xz applet will work just as well as the binary from xz-utils. + + - Thus minimum RAM requirement is 128 MB (physical, and another + 64 to 128 MB using swap) to serve heavily compressed zim files + from AVM Fritz!Box routers, i.e. it may not work with 7270 and + older devices diff --git a/README.md b/README.md new file mode 100644 index 0000000..e8e816d --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ + +This repo is DEPRECATED and here for reference purpose only, it may be deleted from github at any time in the future. + +The clone was originally done with the intention +of integration into freetz, thus the content of +the few commits in this repo has basically moved +to +https://github.com/cm8/freetz/commits/make/zimhttpserver + +Due to the way freetz packages work, there is not +a full clone - make/zimhttpserver/zimhttpserver.mk +rather refers to the original upstream repo, +instead of cloning it fully into freetz. The +reason for freetz to do it this way is, that +it is unclear if a clone is needed until the +user has configured freetz build system. + +After some grace period, /this/ fork will probably +be deleted, because the original intention of +integration into freetz is now finished. If you +have bookmarked it for some reason, take a look +at +https://github.com/cm8/freetz/commit/a89bb115d2d9ec47f7ad85dc3ec24a286b6b1a64 + +This commit basically squashes the changes done +to this repo since fork time. diff --git a/zimHttpServer.pl b/zimHttpServer.pl index af32799..a312729 100755 --- a/zimHttpServer.pl +++ b/zimHttpServer.pl @@ -10,18 +10,24 @@ use strict; use Socket; -my %article; +use List::Util qw( min max ); -my $UNAME=`uname -m`; -chomp($UNAME); -print "UNAME = [$UNAME]\n"; +my %article; my $XZ; -if (-e "./$UNAME/xz") { - $XZ="./$UNAME/xz"; - print "Detected $UNAME, using local xz binary at $XZ\n"; -} else { - print "Using system xz binary\n"; +if (-e "/usr/bin/xz" or -e "/bin/xz") { $XZ="xz"; + print "Using system xz binary.\n"; +} else { + my $UNAME=`uname -m`; + chomp($UNAME); + print "UNAME = [$UNAME]\n"; + + if (-e "./$UNAME/xz") { + $XZ="./$UNAME/xz"; + print "Using local xz binary at $XZ\n"; + } else { + die "No suitable xz decompressor found\n"; + } } # open «file.zim». For more information see internet «openzim.org» @@ -73,17 +79,17 @@ sub xseek { my %header; xseek(\*FILE, 0, 0); # no necesary because it must be it xread(\*FILE, $_, 4); $header{"magicNumber"} = unpack("c*"); # ZIM\x04 -xread(\*FILE, $_, 4); $header{"version"} = unpack("I"); +xread(\*FILE, $_, 4); $header{"version"} = unpack("V"); xread(\*FILE, $_, 16); $header{"uuid"} = unpack("H*"); -xread(\*FILE, $_, 4); $header{"articleCount"} = unpack("I"); -xread(\*FILE, $_, 4); $header{"clusterCount"} = unpack("I"); - -xread(\*FILE, $_, 4); $header{"urlPtrPos"} = unpack("I"); xread(\*FILE, $_, 4); $header{"zero1"} = unpack("I"); -xread(\*FILE, $_, 4); $header{"titlePtrPos"} = unpack("I"); xread(\*FILE, $_, 4); $header{"zero2"} = unpack("I"); -xread(\*FILE, $_, 4); $header{"clusterPtrPos"} = unpack("I"); xread(\*FILE, $_, 4); $header{"zero3"} = unpack("I"); -xread(\*FILE, $_, 4); $header{"mimeListPos"} = unpack("I"); xread(\*FILE, $_, 4); $header{"zero4"} = unpack("I"); -xread(\*FILE, $_, 4); $header{"mainPage"} = unpack("I"); -xread(\*FILE, $_, 4); $header{"layoutPage"} = unpack("I"); +xread(\*FILE, $_, 4); $header{"articleCount"} = unpack("V"); +xread(\*FILE, $_, 4); $header{"clusterCount"} = unpack("V"); + +xread(\*FILE, $_, 4); $header{"urlPtrPos"} = unpack("V"); xread(\*FILE, $_, 4); $header{"zero1"} = unpack("V"); +xread(\*FILE, $_, 4); $header{"titlePtrPos"} = unpack("V"); xread(\*FILE, $_, 4); $header{"zero2"} = unpack("V"); +xread(\*FILE, $_, 4); $header{"clusterPtrPos"} = unpack("V"); xread(\*FILE, $_, 4); $header{"zero3"} = unpack("V"); +xread(\*FILE, $_, 4); $header{"mimeListPos"} = unpack("V"); xread(\*FILE, $_, 4); $header{"zero4"} = unpack("V"); +xread(\*FILE, $_, 4); $header{"mainPage"} = unpack("V"); +xread(\*FILE, $_, 4); $header{"layoutPage"} = unpack("V"); xread(\*FILE, $_, 8); $header{"checksumPos"} = unpack("H*"); # Check that none of the *PtrPos values exceed 32-bits @@ -185,8 +191,8 @@ sub url_pointer{ my $pos = $header{"urlPtrPos"}; $pos += $article*8; xseek(\*FILE, $pos, 0); - xread(\*FILE, $_, 4); my $ret = unpack("I"); - xread(\*FILE, $_, 4); my $ret64 = unpack("I"); + xread(\*FILE, $_, 4); my $ret = unpack("V"); + xread(\*FILE, $_, 4); my $ret64 = unpack("V"); return ($ret64, $ret); } @@ -200,7 +206,7 @@ sub title_pointer{ my $pos = $header{"titlePtrPos"}; $pos += $article_by_title*4; xseek(\*FILE, $pos,0); - xread(\*FILE, $_, 4); my $ret = unpack("I"); + xread(\*FILE, $_, 4); my $ret = unpack("V"); return $ret; } @@ -221,15 +227,17 @@ sub entry{ } xseek(\*FILE, $pos,1); - xread(\*FILE, $_, 2); $article{"mimetype"} = unpack("s"); + # use this to read the field with perl versions < 5.10 + # xread(\*FILE, $_, 2); $article{"mimetype"} = unpack("s", pack("S", unpack("v"))); + xread(\*FILE, $_, 2); $article{"mimetype"} = unpack("s<"); xread(\*FILE, $_, 1); $article{"parameter_len"} = unpack("H*"); xread(\*FILE, $_, 1); $article{"namespace"} = unpack("a"); - xread(\*FILE, $_, 4); $article{"revision"} = unpack("I"); + xread(\*FILE, $_, 4); $article{"revision"} = unpack("V"); if($article{"mimetype"} <0){ - xread(\*FILE, $_, 4); $article{"redirect_index"} = unpack("I"); + xread(\*FILE, $_, 4); $article{"redirect_index"} = unpack("V"); }else{ - xread(\*FILE, $_, 4); $article{"cluster_number"} = unpack("I"); - xread(\*FILE, $_, 4); $article{"blob_number"} = unpack("I"); + xread(\*FILE, $_, 4); $article{"cluster_number"} = unpack("V"); + xread(\*FILE, $_, 4); $article{"blob_number"} = unpack("V"); } $article{"url"} = get_null_string(); $article{"title"} = get_null_string(); @@ -253,8 +261,8 @@ sub cluster_pointer{ my $pos = $header{"clusterPtrPos"}; $pos += $cluster*8; xseek(\*FILE, $pos,0); - xread(\*FILE, $_, 4); my $ret = unpack("I"); - xread(\*FILE, $_, 4); my $ret64 = unpack("I"); + xread(\*FILE, $_, 4); my $ret = unpack("V"); + xread(\*FILE, $_, 4); my $ret64 = unpack("V"); return ($ret64, $ret); } @@ -297,30 +305,49 @@ sub cluster_blob{ if($cluster{"compression_type"} == 4){ my $data_compressed; xread(\*FILE, $data_compressed, $size); + + # The following line breaks because it includes the absolute path of arg0 + #my $file = "/tmp/$ARGV[0]_cluster$cluster-pid$$"; my $file = "/tmp/$$-cluster-$cluster"; -# The following line breaks because it includes the absolute path of arg0 -# my $file = "/tmp/$ARGV[0]_cluster$cluster-pid$$"; + open(DATA, ">$file.xz"); print DATA $data_compressed; close(DATA); - `$XZ -d -f $file.xz`; - open(DATA, "$file"); -# my $blob1; -# xread(DATA, $blob1, 4); -# my $blob_count = int($blob1/4); - seek(DATA, $blob*4, 0); - read(DATA, $_, 4); my $posStart = unpack("I"); - read(DATA, $_, 4); my $posEnd = unpack("I"); - seek(DATA, $posStart, 0); - read(DATA, $ret, $posEnd-$posStart); + + open(DATA, "-|", "$XZ -d -c $file.xz"); + + my $px = $blob * 4; + my $p = 0; + $p += sysread(DATA, $_, min(4*1024, $px-$p)), while $p < $px; + + $px += 4; + $p += sysread(DATA, $_, 4); + my $posStart = unpack("V"); + + $px += 4; + $p += sysread(DATA, $_, 4); + my $posEnd = unpack("V"); + + $px += ($posStart - $px); + $p += sysread(DATA, $_, min(4*1024, $px-$p)), while $p < $px; + + $ret = "x" x (124*1024); + $ret = ""; + $px += ($posEnd - $posStart); + while( $p < $px ) { + $p += sysread(DATA, $_, min(4*1024, $px-$p)); + $ret .= $_; + } + close(DATA); - `rm $file`; + `rm $file.xz`; + return $ret; } elsif ($cluster{"compression_type"} == 1) { my $data; xread(\*FILE, $data, $size); - $_ = substr $data, $blob*4, 4;my $posStart = unpack("I"); - $_ = substr $data, $blob*4+4, 4;my $posEnd = unpack("I"); + $_ = substr $data, $blob*4, 4;my $posStart = unpack("V"); + $_ = substr $data, $blob*4+4, 4;my $posEnd = unpack("V"); $ret = substr $data, $posStart, $posEnd-$posStart; return $ret; } else {