Skip to content
This repository has been archived by the owner on Oct 15, 2024. It is now read-only.

Commit

Permalink
change detection
Browse files Browse the repository at this point in the history 
change detection
update affected .det files
  • Loading branch information
@kaitz
kaitz committed Jan 30, 2020
1 parent 03a7d61 commit 22624cd
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 61 deletions.
4 changes: 2 additions & 2 deletions bmp1.det
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,10 @@ int detect(int c4,int i) {

if (type==BMPFILE && state==START) {
state=INFO;
jend=info*y;
jend=info*y+jstart;
return state;
}
if (type==BMPFILE && state==INFO && i>=jend) {
if (type==BMPFILE && state==INFO && i==(jend-1)) {
state=END;
type=DEFAULT;
bmp=0;
Expand Down
4 changes: 2 additions & 2 deletions bmp4.det
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,10 @@ int detect(int c4,int i) {

if (type==BMPFILE && state==START) {
state=INFO;
jend=info*y;
jend=info*y+jstart;
return state;
}
if (type==BMPFILE && state==INFO && i>=jend) {
if (type==BMPFILE && state==INFO && i==(jend-1)) {
state=END;
type=DEFAULT;
bmp=0;
Expand Down
4 changes: 2 additions & 2 deletions bmp8.det
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,10 @@ int detect(int c4,int i) {

if (type==BMPFILE && state==START) {
state=INFO;
jend=info*y;
jend=info*y+jstart;
return state;
}
if (type==BMPFILE && state==INFO && i>=jend) {
if (type==BMPFILE && state==INFO && i==(jend-1)) {
state=END;
type=DEFAULT;
bmp=0;
Expand Down
10 changes: 5 additions & 5 deletions gif.det
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ int detect(int c4,int i) {
else if (!w) gif=2,gifi=i+3;
else {
type=GIF;
state=INFO;
state=START;
info=w;
jstart=a-1;
jend=i-a+2;
jend=i+1;
return state;
}
}
Expand All @@ -66,11 +66,11 @@ int detect(int c4,int i) {
}
}

if (type==GIF && state==INFO) {
state=START;
if (type==GIF && state==START) {
state=INFO;
return state;
}
if (type==GIF && state==START && i==jend) {
if (type==GIF && state==INFO ) {//&& i==jend-1
state=END;
type=DEFAULT;
gif=0;
Expand Down
42 changes: 19 additions & 23 deletions jpeg.det
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,17 @@
// For JPEG detection
int soi, sof, sos, app,eoi;
int soi, sof, sos, app,eoi;
int type,state,jstart,jend;
int buf0,buf1;
enum {DEFAULT=1,JPEG}; //internal enum
enum {NONE=0,START,INFO,END,RESET=0xfffffffe,REQUEST=0xffffffff}; //external enum
// function will report its state
// function will report its state
// or if i=-1 then state results otherwise i is pos
// c4 is last 4 bytes
void reset(){
state=NONE,type=DEFAULT,soi=sof=sos=app=eoi=jstart=jend=buf0=buf1=0;
state=NONE,type=DEFAULT,soi=sof=sos=app=eoi=jstart=jend=buf0=buf1=0;
}
int detect(int c4,int i) {
int c1;

//if state parameters recuested
if (i==REQUEST){
//printf("DETECT: State requested %d\n",state);
Expand All @@ -21,17 +20,17 @@ int detect(int c4,int i) {
if (state==END) return jend;
if (state==INFO) return 0xffffffff;
}
if (i==RESET) {
reset();
return 0xffffffff;
}
if (i==RESET) {
reset();
return 0xffffffff;
}
c1=c4&255;
buf1=(buf1<<8)|(buf0>>24);
buf0=c4;//buf0<<8|c1;
//detect header
if (!soi && i>=3 && ((
((c4&0xffffff00)==0xffd8ff00 && (c1==0xC0 || c1==0xC4 || (c1>=0xDB && c1<=0xFE)))
||(c4&0xfffffff0)==0xffd8ffe0 ) )
||(c4&0xfffffff0)==0xffd8ffe0 ) )
) soi=i, app=i+2, sos=sof=0;
if (soi) {
if (app==i && (c4>>24)==0xff &&
Expand All @@ -42,10 +41,9 @@ int detect(int c4,int i) {
sos=i;
if (type!=JPEG){
type=JPEG;
state=START;
jstart=soi-3;
soi=buf0=buf1=0;
//printf("DETECT: JPEG start %d\n",jstart);
state=START;
jstart=soi-3;
soi=buf0=buf1=0;
return state;
}
}
Expand All @@ -55,20 +53,18 @@ int detect(int c4,int i) {
eoi=i;
}
if (type==JPEG && soi && sos && eoi && (c4&0xffff)==0xffd8) {
state=END;
type=DEFAULT;
jend=i;//-1;
soi=buf0=buf1=0;
//printf("DETECT: JPEG end-1 %d\n",jend);
state=END;
type=DEFAULT;
jend=i;
soi=buf0=buf1=0;
return state;
}
if (type==JPEG && sos && i>sos && (c4&0xff00)==0xff00
&& (c4&0xff)!=0 && ((c4&0xf8)!=0xd0 )) {
state=END;
type=DEFAULT;
jend=i+1;
soi=buf0=buf1=0;
// printf("DETECT: JPEG end %d\n",jend);
state=END;
type=DEFAULT;
jend=i+1;
soi=buf0=buf1=0;
return state;
}
return NONE;
Expand Down
129 changes: 102 additions & 27 deletions paq8pxv.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2956,6 +2956,11 @@ struct vType {
char encode[16]; // model for encode
int ensize; // size of above model, -1 if no model
int used;
int state; // state of current detection
int start; // start pos of type data in block
int end; // end pos of type data in block
int info; // info of the block if present
int rpos; // pos where start was set in block
};
Array<vStream> vStreams(0);
Array<vType> vTypes(0);
Expand All @@ -2964,23 +2969,31 @@ VM **vmDetect;
VM **vmEncode;
VM **vmDecode;
VM **vmStream;
enum {NONE=0,START,INFO,END,RESET=0xfffffffe,REQUEST=0xffffffff};
enum {NONE=0,START,INFO,END,DISABLE=0xfffffffd,RESET=0xfffffffe,REQUEST=0xffffffff};

// Detect multiple different similar types.
// Change default retorted type to last normal type if recursion type found
// and report it only if it fits to min size of that type.
// If two conflicting detections are found disable first type that reported end state

int detect(File* in, U64 n, int type, int &info, int &info2, int it=0,int s1=0) {
U32 buf0=0; // last 8 bytes
U64 start= in->curpos();
info=-1;
static int deth=0,detd=0; // detected header/data size in bytes
static int dett; // detected block type
if (deth >1) return in->setpos(start+deth),deth=0,dett;
else if (deth ==-1) return in->setpos(start),deth=0,dett;
else if (detd) return in->setpos( start+detd),detd=0,defaultType;
int dstate=0;
info=-1;
static int foundblock=-1;
//int dstate=0;
if (foundblock >-1) {
// report type and set to default type
info=vTypes[foundblock].info;
in->setpos(start+vTypes[foundblock].end);
foundblock=-1;
return defaultType;
}
for (int j=0;j<vTypes.size();j++){
if ( vTypes[j].dsize!=-1){
//reset states
dstate=vmDetect[j]->detect(0,RESET);
if (dstate!=-1 ){ }
vTypes[j].state=NONE;
vmDetect[j]->detect(0,RESET);
}
}
for (U64 i=0; i<n; ++i) {
Expand All @@ -2989,28 +3002,84 @@ info=-1;
buf0=buf0<<8|c;

for (int j=0;j<vTypes.size();j++){
if ( vTypes[j].dsize!=-1){
if (vTypes[j].dsize!=-1 && vTypes[j].state!=DISABLE){
//open type detection file and load into memory
dstate=vmDetect[j]->detect(buf0,i);
if (dstate==START && type==defaultType){ //start vTypes[j].type
//start
int dstate=vmDetect[j]->detect(buf0,i);
if (dstate==START && type==defaultType){
//printf("T=%d START\n",j);
//request current state data
int jst=vmDetect[j]->detect(buf0,REQUEST);
return in->setpos(start+jst), j;// rturn array index for now // vTypes[j].type;
vTypes[j].state=START;
vTypes[j].start=jst; // save type start pos
vTypes[j].rpos=i; // save relative pos
}
if (dstate==INFO){ //info
info=vmDetect[j]->detect(buf0,REQUEST);
// expect START state next
else if (dstate==INFO){
vTypes[j].state=INFO;
vTypes[j].info=vmDetect[j]->detect(buf0,REQUEST);
//printf("T=%d INFO %d\n",j,vTypes[j].info);
}
if (dstate==END){ //end
//request current state data

else if (dstate==END){
// printf("T=%d END\n",j);
// request current state data
vTypes[j].state=END;
foundblock=j;
int jst=vmDetect[j]->detect(buf0,REQUEST);
return in->setpos( start+jst),defaultType;
vTypes[j].end=jst-vTypes[j].start; // save type end pos
}
}
}
if (foundblock >-1) {
bool isrecursionType=false;
// look for active recursive type
for (int j=0;j<vTypes.size();j++){
if (vTypes[j].type<defaultType && vTypes[j].dsize!=-1 && (vTypes[j].state==END)){
isrecursionType=true;
foundblock=j;
break;
}
}
// search for type that still does detection
for (int j=0;j<vTypes.size();j++){
if (isrecursionType==true && vTypes[j].type>defaultType){
//disable nonrecursive type
if ((vTypes[j].state==START || vTypes[j].state==INFO)){ //return acive type
// printf("Type %d s=%d e=%d \n",foundblock,vTypes[foundblock].start, vTypes[foundblock].end);
// printf("Type %d s=%d e=%d r=%d \n",j,vTypes[j].start, vTypes[j].end , vTypes[j].rpos);
if (vTypes[foundblock].start>vTypes[j].rpos){
// if have we real block with good size
// reset pos and set non-default type, restart
foundblock=-1;
return in->setpos(start+vTypes[j].start), j;
}
}
vTypes[j].state=DISABLE;
// printf("T=%d DISABLE NON-RECURSIVE\n",j);
}
else if (vTypes[j].type>defaultType && vTypes[j].dsize!=-1 &&
(vTypes[j].state==START || vTypes[j].state==INFO) && (j!=foundblock)){
vTypes[foundblock].state=DISABLE;
//printf("T=%d DISABLE TYPE\n",j);
foundblock=-1;
break;
}
}

if (foundblock ==-1) continue;
// if single full block then report back
// printf("s=%d e=%d \n",vTypes[foundblock].start, vTypes[foundblock].end);
return in->setpos(start+vTypes[foundblock].start), foundblock;
}
}
for (int j=0;j<vTypes.size();j++){
if ( vTypes[j].state==START || vTypes[j].state==INFO){
foundblock=j;
vTypes[j].end=n-vTypes[j].start;
vTypes[j].state=END;
//printf("s=%d e=%d \n",vTypes[j].start, vTypes[j].end);
return in->setpos(start+vTypes[j].start), j;
}
}

return type;
}

Expand Down Expand Up @@ -3506,12 +3575,16 @@ void compressStream(int streamid,U64 size, File* in, File* out) {
for (int ii=0; ii<8; ii++) datasegmentlen<<=8,datasegmentlen+=segment(datasegmentpos++);
for (int ii=0; ii<4; ii++) datasegmentinfo=(datasegmentinfo<<8)+segment(datasegmentpos++);
if (vTypes[datasegmenttype].type<defaultType || !(isstreamtype(datasegmenttype,i)))datasegmentlen=0;
//printf("Len %d Info %d \n",(U32)datasegmentlen,datasegmentinfo);
if (level>0){
threadencode->predictor->x.filetype=datasegmenttype;
threadencode->predictor->x.blpos=0;
threadencode->predictor->x.finfo=datasegmentinfo;
if (datasegmentlen){
threadencode->predictor->set();
threadencode->predictor->setdebug(0);
}

}
}
for (U64 k=0; k<datasegmentlen; ++k) {
Expand Down Expand Up @@ -4541,11 +4614,13 @@ printf("\n");
//skip if type is recursive or not in current stream
if (vTypes[datasegmenttype].type<defaultType || !(isstreamtype(datasegmenttype,i)))datasegmentlen=0;
if (level>0) {
defaultencoder->predictor->x.filetype=datasegmenttype;
defaultencoder->predictor->x.blpos=0;
defaultencoder->predictor->x.finfo=datasegmentinfo;
defaultencoder->predictor->set();
defaultencoder->predictor->setdebug(0);
defaultencoder->predictor->x.filetype=datasegmenttype;
defaultencoder->predictor->x.blpos=0;
defaultencoder->predictor->x.finfo=datasegmentinfo;
if (datasegmentlen){
defaultencoder->predictor->set();
defaultencoder->predictor->setdebug(0);
}
}
}
for (U64 k=0; k<datasegmentlen; ++k) {
Expand Down

0 comments on commit 22624cd

Please sign in to comment.