From 22624cda139ec277d92b29c4372a1a8facd4d5f2 Mon Sep 17 00:00:00 2001 From: kaitz Date: Thu, 30 Jan 2020 21:55:23 +0200 Subject: [PATCH] change detection change detection update affected .det files --- bmp1.det | 4 +- bmp4.det | 4 +- bmp8.det | 4 +- gif.det | 10 ++-- jpeg.det | 42 ++++++++--------- paq8pxv.cpp | 129 +++++++++++++++++++++++++++++++++++++++++----------- 6 files changed, 132 insertions(+), 61 deletions(-) diff --git a/bmp1.det b/bmp1.det index 5dbeaec..b1eea52 100644 --- a/bmp1.det +++ b/bmp1.det @@ -90,10 +90,10 @@ int detect(int c4,int i) { if (type==BMPFILE && state==START) { state=INFO; - jend=info*y; + jend=info*y+jstart; return state; } - if (type==BMPFILE && state==INFO && i>=jend) { + if (type==BMPFILE && state==INFO && i==(jend-1)) { state=END; type=DEFAULT; bmp=0; diff --git a/bmp4.det b/bmp4.det index ca3f785..ba4062e 100644 --- a/bmp4.det +++ b/bmp4.det @@ -90,10 +90,10 @@ int detect(int c4,int i) { if (type==BMPFILE && state==START) { state=INFO; - jend=info*y; + jend=info*y+jstart; return state; } - if (type==BMPFILE && state==INFO && i>=jend) { + if (type==BMPFILE && state==INFO && i==(jend-1)) { state=END; type=DEFAULT; bmp=0; diff --git a/bmp8.det b/bmp8.det index e15565f..d4dd62b 100644 --- a/bmp8.det +++ b/bmp8.det @@ -89,10 +89,10 @@ int detect(int c4,int i) { if (type==BMPFILE && state==START) { state=INFO; - jend=info*y; + jend=info*y+jstart; return state; } - if (type==BMPFILE && state==INFO && i>=jend) { + if (type==BMPFILE && state==INFO && i==(jend-1)) { state=END; type=DEFAULT; bmp=0; diff --git a/gif.det b/gif.det index 23e2d5c..2dedfb0 100644 --- a/gif.det +++ b/gif.det @@ -54,10 +54,10 @@ int detect(int c4,int i) { else if (!w) gif=2,gifi=i+3; else { type=GIF; - state=INFO; + state=START; info=w; jstart=a-1; - jend=i-a+2; + jend=i+1; return state; } } @@ -66,11 +66,11 @@ int detect(int c4,int i) { } } - if (type==GIF && state==INFO) { - state=START; + if (type==GIF && state==START) { + state=INFO; return state; } - if (type==GIF && state==START && i==jend) { + if (type==GIF && state==INFO ) {//&& i==jend-1 state=END; type=DEFAULT; gif=0; diff --git a/jpeg.det b/jpeg.det index 219b15b..e480951 100644 --- a/jpeg.det +++ b/jpeg.det @@ -1,18 +1,17 @@ // For JPEG detection -int soi, sof, sos, app,eoi; +int soi, sof, sos, app,eoi; int type,state,jstart,jend; int buf0,buf1; enum {DEFAULT=1,JPEG}; //internal enum enum {NONE=0,START,INFO,END,RESET=0xfffffffe,REQUEST=0xffffffff}; //external enum -// function will report its state +// function will report its state // or if i=-1 then state results otherwise i is pos // c4 is last 4 bytes void reset(){ - state=NONE,type=DEFAULT,soi=sof=sos=app=eoi=jstart=jend=buf0=buf1=0; + state=NONE,type=DEFAULT,soi=sof=sos=app=eoi=jstart=jend=buf0=buf1=0; } int detect(int c4,int i) { int c1; - //if state parameters recuested if (i==REQUEST){ //printf("DETECT: State requested %d\n",state); @@ -21,17 +20,17 @@ int detect(int c4,int i) { if (state==END) return jend; if (state==INFO) return 0xffffffff; } - if (i==RESET) { - reset(); - return 0xffffffff; - } + if (i==RESET) { + reset(); + return 0xffffffff; + } c1=c4&255; buf1=(buf1<<8)|(buf0>>24); buf0=c4;//buf0<<8|c1; //detect header if (!soi && i>=3 && (( ((c4&0xffffff00)==0xffd8ff00 && (c1==0xC0 || c1==0xC4 || (c1>=0xDB && c1<=0xFE))) - ||(c4&0xfffffff0)==0xffd8ffe0 ) ) + ||(c4&0xfffffff0)==0xffd8ffe0 ) ) ) soi=i, app=i+2, sos=sof=0; if (soi) { if (app==i && (c4>>24)==0xff && @@ -42,10 +41,9 @@ int detect(int c4,int i) { sos=i; if (type!=JPEG){ type=JPEG; - state=START; - jstart=soi-3; - soi=buf0=buf1=0; - //printf("DETECT: JPEG start %d\n",jstart); + state=START; + jstart=soi-3; + soi=buf0=buf1=0; return state; } } @@ -55,20 +53,18 @@ int detect(int c4,int i) { eoi=i; } if (type==JPEG && soi && sos && eoi && (c4&0xffff)==0xffd8) { - state=END; - type=DEFAULT; - jend=i;//-1; - soi=buf0=buf1=0; - //printf("DETECT: JPEG end-1 %d\n",jend); + state=END; + type=DEFAULT; + jend=i; + soi=buf0=buf1=0; return state; } if (type==JPEG && sos && i>sos && (c4&0xff00)==0xff00 && (c4&0xff)!=0 && ((c4&0xf8)!=0xd0 )) { - state=END; - type=DEFAULT; - jend=i+1; - soi=buf0=buf1=0; - // printf("DETECT: JPEG end %d\n",jend); + state=END; + type=DEFAULT; + jend=i+1; + soi=buf0=buf1=0; return state; } return NONE; diff --git a/paq8pxv.cpp b/paq8pxv.cpp index b867d49..814282d 100644 --- a/paq8pxv.cpp +++ b/paq8pxv.cpp @@ -2956,6 +2956,11 @@ struct vType { char encode[16]; // model for encode int ensize; // size of above model, -1 if no model int used; + int state; // state of current detection + int start; // start pos of type data in block + int end; // end pos of type data in block + int info; // info of the block if present + int rpos; // pos where start was set in block }; Array vStreams(0); Array vTypes(0); @@ -2964,23 +2969,31 @@ VM **vmDetect; VM **vmEncode; VM **vmDecode; VM **vmStream; -enum {NONE=0,START,INFO,END,RESET=0xfffffffe,REQUEST=0xffffffff}; +enum {NONE=0,START,INFO,END,DISABLE=0xfffffffd,RESET=0xfffffffe,REQUEST=0xffffffff}; + +// Detect multiple different similar types. +// Change default retorted type to last normal type if recursion type found +// and report it only if it fits to min size of that type. +// If two conflicting detections are found disable first type that reported end state int detect(File* in, U64 n, int type, int &info, int &info2, int it=0,int s1=0) { U32 buf0=0; // last 8 bytes U64 start= in->curpos(); -info=-1; - static int deth=0,detd=0; // detected header/data size in bytes - static int dett; // detected block type - if (deth >1) return in->setpos(start+deth),deth=0,dett; - else if (deth ==-1) return in->setpos(start),deth=0,dett; - else if (detd) return in->setpos( start+detd),detd=0,defaultType; - int dstate=0; + info=-1; + static int foundblock=-1; + //int dstate=0; + if (foundblock >-1) { + // report type and set to default type + info=vTypes[foundblock].info; + in->setpos(start+vTypes[foundblock].end); + foundblock=-1; + return defaultType; + } for (int j=0;jdetect(0,RESET); - if (dstate!=-1 ){ } + vTypes[j].state=NONE; + vmDetect[j]->detect(0,RESET); } } for (U64 i=0; idetect(buf0,i); - if (dstate==START && type==defaultType){ //start vTypes[j].type - //start + int dstate=vmDetect[j]->detect(buf0,i); + if (dstate==START && type==defaultType){ + //printf("T=%d START\n",j); //request current state data int jst=vmDetect[j]->detect(buf0,REQUEST); - return in->setpos(start+jst), j;// rturn array index for now // vTypes[j].type; + vTypes[j].state=START; + vTypes[j].start=jst; // save type start pos + vTypes[j].rpos=i; // save relative pos } - if (dstate==INFO){ //info - info=vmDetect[j]->detect(buf0,REQUEST); - // expect START state next + else if (dstate==INFO){ + vTypes[j].state=INFO; + vTypes[j].info=vmDetect[j]->detect(buf0,REQUEST); + //printf("T=%d INFO %d\n",j,vTypes[j].info); } - if (dstate==END){ //end - //request current state data - + else if (dstate==END){ + // printf("T=%d END\n",j); + // request current state data + vTypes[j].state=END; + foundblock=j; int jst=vmDetect[j]->detect(buf0,REQUEST); - return in->setpos( start+jst),defaultType; + vTypes[j].end=jst-vTypes[j].start; // save type end pos } } } + if (foundblock >-1) { + bool isrecursionType=false; + // look for active recursive type + for (int j=0;jdefaultType){ + //disable nonrecursive type + if ((vTypes[j].state==START || vTypes[j].state==INFO)){ //return acive type + // printf("Type %d s=%d e=%d \n",foundblock,vTypes[foundblock].start, vTypes[foundblock].end); + // printf("Type %d s=%d e=%d r=%d \n",j,vTypes[j].start, vTypes[j].end , vTypes[j].rpos); + if (vTypes[foundblock].start>vTypes[j].rpos){ + // if have we real block with good size + // reset pos and set non-default type, restart + foundblock=-1; + return in->setpos(start+vTypes[j].start), j; + } + } + vTypes[j].state=DISABLE; + // printf("T=%d DISABLE NON-RECURSIVE\n",j); + } + else if (vTypes[j].type>defaultType && vTypes[j].dsize!=-1 && + (vTypes[j].state==START || vTypes[j].state==INFO) && (j!=foundblock)){ + vTypes[foundblock].state=DISABLE; + //printf("T=%d DISABLE TYPE\n",j); + foundblock=-1; + break; + } + } + + if (foundblock ==-1) continue; + // if single full block then report back + // printf("s=%d e=%d \n",vTypes[foundblock].start, vTypes[foundblock].end); + return in->setpos(start+vTypes[foundblock].start), foundblock; + } } + for (int j=0;jsetpos(start+vTypes[j].start), j; + } + } + return type; } @@ -3506,12 +3575,16 @@ void compressStream(int streamid,U64 size, File* in, File* out) { for (int ii=0; ii<8; ii++) datasegmentlen<<=8,datasegmentlen+=segment(datasegmentpos++); for (int ii=0; ii<4; ii++) datasegmentinfo=(datasegmentinfo<<8)+segment(datasegmentpos++); if (vTypes[datasegmenttype].type0){ threadencode->predictor->x.filetype=datasegmenttype; threadencode->predictor->x.blpos=0; threadencode->predictor->x.finfo=datasegmentinfo; + if (datasegmentlen){ threadencode->predictor->set(); threadencode->predictor->setdebug(0); + } + } } for (U64 k=0; k0) { - defaultencoder->predictor->x.filetype=datasegmenttype; - defaultencoder->predictor->x.blpos=0; - defaultencoder->predictor->x.finfo=datasegmentinfo; - defaultencoder->predictor->set(); - defaultencoder->predictor->setdebug(0); + defaultencoder->predictor->x.filetype=datasegmenttype; + defaultencoder->predictor->x.blpos=0; + defaultencoder->predictor->x.finfo=datasegmentinfo; + if (datasegmentlen){ + defaultencoder->predictor->set(); + defaultencoder->predictor->setdebug(0); + } } } for (U64 k=0; k