From 2dd036fc9f8144d6c8b17f3097cdbb97b8d09b7d Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Sat, 26 Mar 2022 13:08:30 +0100 Subject: [PATCH] Release 1.6.24 --- ChangeLog | 1216 ++++++++++++++++++++++++----------------------- README.md | 495 ++++++++----------- bin/Makefile.am | 2 +- configure.ac | 2 +- 4 files changed, 824 insertions(+), 891 deletions(-) diff --git a/ChangeLog b/ChangeLog index d261c96d..a885c493 100755 --- a/ChangeLog +++ b/ChangeLog @@ -1,810 +1,826 @@ 2022-03-26 -- Update Makefile.am with ACLOCAL_AMFLAGS: #336 -- Update Doxigen.in file: #332 + +- Release 1.6.24 + +2022-03-26 + +- Update m4 files +- Update Makefile.am with ACLOCAL_AMFLAGS: #336 +- Update Doxigen.in file: #332 2022-03-11 -- Fix cmd line processinf in nfanon. #328 + +- Fix cmd line processinf in nfanon. #328 2022-03-03 -- Make configure.ac autoconf 2.69 compatible + +- Make configure.ac autoconf 2.69 compatible 2022-02-25 -- Cleanup automake files. Fixes #304. + +- Cleanup automake files. Fixes #304. 2021-10-22 -- Fix link handling in nfpcapd + +- Fix link handling in nfpcapd 2021-08-27 -- Fix compile flags #304 + +- Fix compile flags #304 2021-07-31 -- Fix nfdump man page #301 -- Fix minor bugs -- Add NAT event record support for IPFIX. #298 + +- Fix nfdump man page #301 +- Fix minor bugs +- Add NAT event record support for IPFIX. #298 2021-07-16 -- Fix issue #296 - broken json format with option -q + +- Fix issue #296 - broken json format with option -q 2021-06-25 -- Fix json msec formating + +- Fix json msec formating 2021-05-19 -- Silence short packet logs due to small snaplen in pcaproc.c #221 + +- Silence short packet logs due to small snaplen in pcaproc.c #221 2021-05-05 -- Release 1.6.23 + +- Release 1.6.23 2021-04-28 -- Fix potential FreeNode without valid Node in nfpcapd. + +- Fix potential FreeNode without valid Node in nfpcapd. 2021-04-21 -- Add all non TCP/UDP IP protocols as streams in nfpcapd + +- Add all non TCP/UDP IP protocols as streams in nfpcapd 2021-04-20 -- Add mpls unwrap in nfpcapd. Skip MPLS labels -- Add ESP to processed protocols in nfpcapd. -- Some Code cleanup + +- Add mpls unwrap in nfpcapd. Skip MPLS labels +- Add ESP to processed protocols in nfpcapd. +- Some Code cleanup 2021-04-10 -- Change spin lock to native C11 lock -- Cleanup code for issue #283 + +- Change spin lock to native C11 lock +- Cleanup code for issue #283 2021-04-09 -- Fix minor nfpcapd issues -- Add mpls unwrap in sflow code - adds mpls labels if available + +- Fix minor nfpcapd issues +- Add mpls unwrap in sflow code - adds mpls labels if available 2021-03-12 -- Update rbtree. -- Fix potential deadlock in nfpcapd if it terminates. + +- Update rbtree. +- Fix potential deadlock in nfpcapd if it terminates. 2021-03-06 -- Add packet capture buffer size to nfpcapd + +- Add packet capture buffer size to nfpcapd 2021-02-20 -- Fix sflow code extended field parsing. #262 and #273 -- Fix endless loop of nfexpire, if it does not find files + +- Fix sflow code extended field parsing. #262 and #273 +- Fix endless loop of nfexpire, if it does not find files 2021-01-30 -- Fix processing deoding error for yaf exporter -- Zero out tcp flags for non TCP records + +- Fix processing deoding error for yaf exporter +- Zero out tcp flags for non TCP records 2021-01-16 -- Add reverse element enterprise ID 29305 for counter values -- Add biFlow direction element 239 -- Add flow end reason element 136 -- Make -Tall the default for nfcapd to collect extensions + +- Add reverse element enterprise ID 29305 for counter values +- Add biFlow direction element 239 +- Add flow end reason element 136 +- Make -Tall the default for nfcapd to collect extensions 2020-12-22 -- Code cleanup and boundary checks in option template processing + +- Code cleanup and boundary checks in option template processing 2020-12-19 -- Implement element 160 (SystemInitTime) in option template + +- Implement element 160 (SystemInitTime) in option template 2020-12-19 -- Add Element 160 (SystemInitTime) in flow record used by Huawei + +- Add Element 160 (SystemInitTime) in flow record used by Huawei 2020-12-05 -- Fix path handling for -l -- Fix print plain numbers #263 + +- Fix path handling for -l +- Fix print plain numbers #263 2020-11-21 -- Release 1.6.22 + +- Release 1.6.22 2020-10-18 -- Fix nfreplay v5 time shift bug + +- Fix nfreplay v5 time shift bug 2020-10-17 -- add support for >=, <= comparators. #256. Thanks to piorek94 -- Fix yacc/bison warnings. Cleanup unused tokens -- Fix syntax error 'flags AS' as AS is a reserved word. #255 -- Add element 139 for ICMP type/code in IPv6. #250 -- Fix IPv4/IPv6 statistics representation #252 + +- add support for >=, <= comparators. #256. Thanks to piorek94 +- Fix yacc/bison warnings. Cleanup unused tokens +- Fix syntax error 'flags AS' as AS is a reserved word. #255 +- Add element 139 for ICMP type/code in IPv6. #250 +- Fix IPv4/IPv6 statistics representation #252 2020-09-12 -- Cleanup nip/xip filter syntax. Add filter syntax 'nip in [ ]'. Request #246 + +- Cleanup nip/xip filter syntax. Add filter syntax 'nip in [ ]'. Request #246 2020-09-03 -- Add nfversion to nfpcapd + +- Add nfversion to nfpcapd 2020-08-31 -- Add collected netflow/sflow version in nfdump record. Request #242 -- Fix GuessDir bug - issue #215 + +- Add collected netflow/sflow version in nfdump record. Request #242 +- Fix GuessDir bug - issue #215 2020-08-02 -- Re-address issue #231 - remove strict rule rfc 7011 + +- Re-address issue #231 - remove strict rule rfc 7011 2020-08-02 -- Release 1.6.21 -- Address issue #159. Implement rfc 7011 and include sender UDP port into unique template identification -- Address issue #236 Add token 'dir' equivalent to 'flowdir' in filter syntax + +- Release 1.6.21 +- Address issue #159. Implement rfc 7011 and include sender UDP port into unique template identification +- Address issue #236 Add token 'dir' equivalent to 'flowdir' in filter syntax 2020-07-25 -- Add optional print direction ascending or descending to output of statistics -s and oredered printing -O. Request #235 -- Fix issue #234 -- Fix #230 - Avoid use_syslog name clash on certain OS + +- Add optional print direction ascending or descending to output of statistics -s and oredered printing -O. Request #235 +- Fix issue #234 +- Fix #230 - Avoid use_syslog name clash on certain OS 2020-06-20 -- Honor -n flag when printing sorted flow cache -- Fix uninitialized variable printPlain -- Fix bug #223 limit matchig flows -c -- Restore old behaviour unlimiting output flows unless in -s stat -- Fix ft2nfdump nexthop fields -- Fix ft2nfdump extension map size -- internal: put output parameters in a single struct -- Fix GuessDir bug - issue #215 -- Compact Changelog -- Fix GuessDir bug - issue #215 in flow exporter + +- Honor -n flag when printing sorted flow cache +- Fix uninitialized variable printPlain +- Fix bug #223 limit matchig flows -c +- Restore old behaviour unlimiting output flows unless in -s stat +- Fix ft2nfdump nexthop fields +- Fix ft2nfdump extension map size +- internal: put output parameters in a single struct +- Fix GuessDir bug - issue #215 +- Compact Changelog +- Fix GuessDir bug - issue #215 in flow exporter 2020-03-29 -- Release 1.6.20 -- More cleanup on plain number printing -- Fix plain numbers bug #213 -- Fix profiler filer bug + +- Release 1.6.20 +- More cleanup on plain number printing +- Fix plain numbers bug #213 +- Fix profiler filer bug 2020-02-22 -- Release 1.6.19 -- Add Source security group tag (SGT 34000) issue #192 -- Modify heuristic for bidir flows #59 -- Add Push for new sgt tag. -- Cleanup unused nffile records. Add nffileV2.h -- Fix various compiler warnings and automake issues -- Remove external global vars -- Set verbose logging off by default in non daemon mode -- Fix make check -- Remove old legacy records -- Major code rearrangement in order to prepare for futur versions of nfdump - Move extension definitions into extension files nfx.h - Move exporter definition into extension.h - nffile.h should only contain nffile data block handlich definitions -- Fix compile error for FreeBSD #203 -- Cleanup header code - add new filter.h - free nfdump.h from dust -- More code cleanup - delete nf_common - add output_fmt -- Fix bug for IPv6 network cidr representation in raw and cvs output mode. -- Remove unused old code. Fix for #197 -- Fix bidir export issues. Cleanup code. Fix for #195 -- Enable small time intervals < 60s. - #185 -- Cleanup old code -- Fix minor nfpcapd issues -- Fix gcc compile issue -- Fix corrupt file handling #193 -- More code cleanup. Move code to dedicated files. -- Restructure code for output into separate files. -- Cleanup old code. -- Fix bug #189 - valid json output -- Fix issue #190. Add compile time option for JunOS. -- Cleanup configure.ac -- Fix various c11 compile issues -- Cleanup old compat15 comments -- Add vlan tags dot1qVlanId, 243, 254 - #182 -- Fix sflow issue with Arista switches -- Fix ft2nfdump next hop and ip router fields -- Cleanup and fix IPv6 network display in war records. -- Fix compile issues -- Fix output buffer size for lzo1x_decompress_safe() -- Fix VerifyExtensionMap #179 -- Fix compile errors + +- Release 1.6.19 +- Add Source security group tag (SGT 34000) issue #192 +- Modify heuristic for bidir flows #59 +- Add Push for new sgt tag. +- Cleanup unused nffile records. Add nffileV2.h +- Fix various compiler warnings and automake issues +- Remove external global vars +- Set verbose logging off by default in non daemon mode +- Fix make check +- Remove old legacy records +- Major code rearrangement in order to prepare for futur versions of nfdump Move extension definitions into extension files nfx.h Move exporter definition into extension.h nffile.h should only contain nffile data block handlich definitions +- Fix compile error for FreeBSD #203 +- Cleanup header code - add new filter.h - free nfdump.h from dust +- More code cleanup - delete nf_common - add output_fmt +- Fix bug for IPv6 network cidr representation in raw and cvs output mode. +- Remove unused old code. Fix for #197 +- Fix bidir export issues. Cleanup code. Fix for #195 +- Enable small time intervals < 60s. - #185 +- Cleanup old code +- Fix minor nfpcapd issues +- Fix gcc compile issue +- Fix corrupt file handling #193 +- More code cleanup. Move code to dedicated files. +- Restructure code for output into separate files. +- Cleanup old code. +- Fix bug #189 - valid json output +- Fix issue #190. Add compile time option for JunOS. +- Cleanup configure.ac +- Fix various c11 compile issues +- Cleanup old compat15 comments +- Add vlan tags dot1qVlanId, 243, 254 - #182 +- Fix sflow issue with Arista switches +- Fix ft2nfdump next hop and ip router fields +- Cleanup and fix IPv6 network display in war records. +- Fix compile issues +- Fix output buffer size for lzo1x_decompress_safe() +- Fix VerifyExtensionMap #179 +- Fix compile errors 2019-08-05 -- Release 1.6.18 -- Fix nfdump.1 man page. #175 -- Fix off by 1 array. #173 -- Fix use after free in ModifyCompressFile -- Add bound checks in AddExporterStat #174 -- Add bound checks in AddSamplerInfo #176 -- Add bound checks in AddExporterInfo -- Fix checks in InsertExtensionMap #177 -- Remove COMPAT15 code - should no longer be needed. -- Merge pull request #167 -- Cleanup old code -- Replace depricated pcap_lookupdev call in nfpcapd + +- Release 1.6.18 +- Fix nfdump.1 man page. #175 +- Fix off by 1 array. #173 +- Fix use after free in ModifyCompressFile +- Add bound checks in AddExporterStat #174 +- Add bound checks in AddSamplerInfo #176 +- Add bound checks in AddExporterInfo +- Fix checks in InsertExtensionMap #177 +- Remove COMPAT15 code - should no longer be needed. +- Merge pull request #167 +- Cleanup old code +- Replace depricated pcap_lookupdev call in nfpcapd 2019-07-31 -- Add early record size sanity check also for nfprofile, nfanon and nfreplay + +- Add early record size sanity check also for nfprofile, nfanon and nfreplay 2019-07-26 -- nfpcapd cleanup, add some more monitoring -- Fix hbo_exporter.c:249_1 segfault -- Fix hbo_nffile_inline.c:85_1 segfault -- Fix hbo_nfx.c:216_3 segfault -- Update minilzo to v2.10 -- Change to safe lzo decompress function + +- nfpcapd cleanup, add some more monitoring +- Fix hbo_exporter.c:249_1 segfault +- Fix hbo_nffile_inline.c:85_1 segfault +- Fix hbo_nfx.c:216_3 segfault +- Update minilzo to v2.10 +- Change to safe lzo decompress function 2019-07-25 -- Rework nfpcapd and add it officially to the nfdump collection. -- Add nfpcapd man page -- Fix potential unsigned integer underflow #171 + +- Rework nfpcapd and add it officially to the nfdump collection. +- Add nfpcapd man page +- Fix potential unsigned integer underflow #171 2019-07-16 -- Add latency extension if dumping flowcache + +- Add latency extension if dumping flowcache 2019-07-15 -- Fix typos -- Fix exporter struct inconsistancies. Coredump on ARM otherwise. + +- Fix typos +- Fix exporter struct inconsistancies. Coredump on ARM otherwise. 2019-07-02 -- Add ipfix element #150, #151 unix time start/end -- Fix display bug raw record + +- Add ipfix element #150, #151 unix time start/end +- Fix display bug raw record 2019-06-01 -- Add ipfix dyn element handling. -- Add empty m4 directory - keep autoconf happy + +- Add ipfix dyn element handling. +- Add empty m4 directory - keep autoconf happy 2019-06-01 -- Fix issue #162 - ipfix mpls sequece. -- Fix issue #156 - print flowtable index error + +- Fix issue #162 - ipfix mpls sequece. +- Fix issue #156 - print flowtable index error 2019-03-17 -- Fix spec file -- Remove non thread safe logging in nfpcapd + +- Fix spec file +- Remove non thread safe logging in nfpcapd 2018-11-24 -- Fix protocol tag for protocol 87 - TCF - #130 -- Add TCP flags ECN,CVR - #132 -- Fix some error messages to be printed to the correct stream #135 -- Add missing -M command line help to nfcapd -- Remove padding byte warning in log #141 -- Fix bug to accept -y compression flag in nfcapd. - #145 + +- Fix protocol tag for protocol 87 - TCF - #130 +- Add TCP flags ECN,CVR - #132 +- Fix some error messages to be printed to the correct stream #135 +- Add missing -M command line help to nfcapd +- Remove padding byte warning in log #141 +- Fix bug to accept -y compression flag in nfcapd. - #145 2018-06-24 -- Fix bookkeeper type - use key_t -- Add multiple packet repeaters to nfcapd/sfcapd. Up to 8 repeaters (-R) can be defined. -- Ignore OSX .DS_Store files in -R file list -- Add CISCO ASA elements initiatorPackets (298) responderPackets (299) -- Merge #120 pull request for -z parameter to nfreplay -- Update man page nfreplay + +- Fix bookkeeper type - use key_t +- Add multiple packet repeaters to nfcapd/sfcapd. Up to 8 repeaters (-R) can be defined. +- Ignore OSX .DS_Store files in -R file list +- Add CISCO ASA elements initiatorPackets (298) responderPackets (299) +- Merge #120 pull request for -z parameter to nfreplay +- Update man page nfreplay 2018-05-06 -- New bookkeeper hash broke NfSen. Fixed. ported back to release 1.6.17 + +- New bookkeeper hash broke NfSen. Fixed. ported back to release 1.6.17 2018-04-20 -- Release 1.6.17 + +- Release 1.6.17 2018-04-20 -- Fix bug in sorting when guessing flow direction. Issue #92 -- Update nfdump.1 man page for xsrcport & xdstport aggregations. Request #109 -- Fix minor bugs -- Fix definition for InfluxDB in configure.ac Issue #98 + +- Fix bug in sorting when guessing flow direction. Issue #92 +- Update nfdump.1 man page for xsrcport & xdstport aggregations. Request #109 +- Fix minor bugs +- Fix definition for InfluxDB in configure.ac Issue #98 2018-04-01 -- Add program exit in nfx.c after panic with correupt data file -- Add missing size check when reading nfdump 1.5.x common record blocks -- Add missing option -M in man page. Issue #103 -- Add Fix processing of influx URL in nfprofile + +- Add program exit in nfx.c after panic with correupt data file +- Add missing size check when reading nfdump 1.5.x common record blocks +- Add missing option -M in man page. Issue #103 +- Add Fix processing of influx URL in nfprofile 2018-02-11 -- Add missing json output format in nfdump help text -- Add missing -v option in nfreplay help text + +- Add missing json output format in nfdump help text +- Add missing -v option in nfreplay help text 2018-01-06 -- Merge pull request #51 Influxdb from Luca. Thx for the patch + +- Merge pull request #51 Influxdb from Luca. Thx for the patch 2018-01-01 -- IPFIX time stamps - Fix elements #21,#22 offset calculation, but timestamps not yet evaluated. (#160) -- IPFIX add fwd status tag #89 compatible to v9 (1byte) + +- IPFIX time stamps - Fix elements #21,#22 offset calculation, but timestamps not yet evaluated. (#160) +- IPFIX add fwd status tag #89 compatible to v9 (1byte) 2017-12-31 -- IPFIX sampling - sampling algorithm no longer required for tag #34 -- IPFIX sampling add tags #305 and #304 - set them identical to #34, #35 + +- IPFIX sampling - sampling algorithm no longer required for tag #34 +- IPFIX sampling add tags #305 and #304 - set them identical to #34, #35 2017-12-30 -- Add new output format json. Print each record as individual json object + +- Add new output format json. Print each record as individual json object 2017-12-28 -- Add sampling elements ID 302,304,305. put them identical to ID 48,49,50 -- Add option to label filter terms. syntax: () %labelname. -- Add %lbl option to print flow label in output -- Update nfdump(1) man page for flowlabels + +- Add sampling elements ID 302,304,305. put them identical to ID 48,49,50 +- Add option to label filter terms. syntax: () %labelname. +- Add %lbl option to print flow label in output +- Update nfdump(1) man page for flowlabels 2017-12-27 -- Add ipfix delta timestamp elements 158/159. -- Update sflow code to commit 7322984 of https://github.com/sflow/sflowtool -- Cleanup sflow code - uncomment unnecessary code -- Fix header includes" -- Fix 64bit fts compat issue in fts_compat.c -- Add more detailed autogen.sh - softlink bootstrap + +- Add ipfix delta timestamp elements 158/159. +- Update sflow code to commit 7322984 of https://github.com/sflow/sflowtool +- Cleanup sflow code - uncomment unnecessary code +- Fix header includes" +- Fix 64bit fts compat issue in fts_compat.c +- Add more detailed autogen.sh - softlink bootstrap 2017-12-22 -- Fix potential memory leaks in nfpcapd + +- Fix potential memory leaks in nfpcapd 2017-12-21 -- Fix wrong offset calculation if unknown options are found -- Add x-late src/dst ip aggregation, if compiled with NSEL support + +- Fix wrong offset calculation if unknown options are found +- Add x-late src/dst ip aggregation, if compiled with NSEL support 2017-12-17 -- Add ipfix sampling. Process option template/record with sampling elements 34 and 35 -- Report updates on existing samplers in v9 only if values change. issue 84 + +- Add ipfix sampling. Process option template/record with sampling elements 34 and 35 +- Report updates on existing samplers in v9 only if values change. issue 84 2017-11-05 v1.6.16 2017-12-10 -- Add lz4 compression -- Remove old xstat legancy code, not needed -- Remove automake files from git + +- Add lz4 compression +- Remove old xstat legancy code, not needed +- Remove automake files from git 2017-12-03 -- Fix old 1.6.15 tags -- Fix minor issues and compiler warnings + +- Fix old 1.6.15 tags +- Fix minor issues and compiler warnings 2017-10-22 -- Add support for CISCO IOS 8 bytes timestamps ID 21/22 -- Fix issue #72 - multiple stat output -- Change -B behaviour as proposed in issue #59. Should not impact with previous use, but is more flexible -- Add bzip compress switch in usage output of nfpcapd -- Fix compile issues on some platforms -- nfpcapd improvements - still beta software. -- Minor bug fixes + +- Add support for CISCO IOS 8 bytes timestamps ID 21/22 +- Fix issue #72 - multiple stat output +- Change -B behaviour as proposed in issue #59. Should not impact with previous use, but is more flexible +- Add bzip compress switch in usage output of nfpcapd +- Fix compile issues on some platforms +- nfpcapd improvements - still beta software. +- Minor bug fixes 2016-11-25 -- Add latency extension to nfpcapd -- Smaller bug fixes to nfpcapd + +- Add latency extension to nfpcapd +- Smaller bug fixes to nfpcapd 2016-07-23 -- Replace unreliable _ftok with more reliable string hash + +- Replace unreliable \_ftok with more reliable string hash 2016-07-20 -- Aggregate using in+out bytes for bidirectional flows + +- Aggregate using in+out bytes for bidirectional flows 2016-06-05 v.1.6.15 -- Fix Security issue http://www.security-assessment.com/files/documents/advisory/Nfdump%20nfcapd%201.6.14%20-%20Multiple%20Vulnerabilities.pdf -- Fix obyte, opps and obps output records -- Fix wrong bps type case in cvs output. Fix opbs ipbs typos + +- Fix Security issue http://www.security-assessment.com/files/documents/advisory/Nfdump%20nfcapd%201.6.14%20-%20Multiple%20Vulnerabilities.pdf +- Fix obyte, opps and obps output records +- Fix wrong bps type case in cvs output. Fix opbs ipbs typos 2016-01-10 v.1.6.14 -- Fix CentOS compile issues with flow-tools converter -- Fix FreeBSD,OpenBSD build problems -- Fix timestamp overflow in sflow.c + +- Fix CentOS compile issues with flow-tools converter +- Fix FreeBSD,OpenBSD build problems +- Fix timestamp overflow in sflow.c 2015-12-23 -- Fix IP Fragmentation in sflow collector -- Create libnfdump for dynamic linking + +- Fix IP Fragmentation in sflow collector +- Create libnfdump for dynamic linking 2015-10-02 -- Fix compile errors on other platforms -- Add -R to ModifyCompression -- Add std sampler ID 4 Bytes and allow random sampler (tag 50) -- Add BZ2 compression along existing LZ0 -- Add direct write to flowtools converter ft2nfdump -- Fix zero alignment bug, if only half of an extension is sent -- Fix nfanon time window bug in subsequent files in -R list -- Fix CommonRecordV0Type conversion bug -- Fix nfexport bug, if only one single map exists + +- Fix compile errors on other platforms +- Add -R to ModifyCompression +- Add std sampler ID 4 Bytes and allow random sampler (tag 50) +- Add BZ2 compression along existing LZ0 +- Add direct write to flowtools converter ft2nfdump +- Fix zero alignment bug, if only half of an extension is sent +- Fix nfanon time window bug in subsequent files in -R list +- Fix CommonRecordV0Type conversion bug +- Fix nfexport bug, if only one single map exists 2014-11-16 v.1.6.13 -- Fix v1 extension size bug -- Add htonll check for autoconf -- Fix AddExtensionMap compare bug -- Fix ipfix templare withdraw problems - free all maps correctly -- Add minilzo 2.08 - fixes CVE-2014-4607 -- Cleanup some stat code. more needs to be done .. -- Cleanup man pages for -O -n -- Remove SunPro test in configure - no longer supported anyway -- Cleanup NAT/NSEL filter differences + +- Fix v1 extension size bug +- Add htonll check for autoconf +- Fix AddExtensionMap compare bug +- Fix ipfix templare withdraw problems - free all maps correctly +- Add minilzo 2.08 - fixes CVE-2014-4607 +- Cleanup some stat code. more needs to be done .. +- Cleanup man pages for -O -n +- Remove SunPro test in configure - no longer supported anyway +- Cleanup NAT/NSEL filter differences 2014-06-15 v1.6.12p1 -- Add pblock compare functions -- Update extended filter: Allow modification left/right values + +- Add pblock compare functions +- Update extended filter: Allow modification left/right values 2014-02-16 v1.6.12 -- Add NAT pool port allocation -- Modify/fix NAT vrf tags. Add egress vrf ID -- Modify common record due to exporter exhaustion. new common record - type 10 adds 4 extra bytes. Reads v1 common record transparently -- Fix sflow potential crash + +- Add NAT pool port allocation +- Modify/fix NAT vrf tags. Add egress vrf ID +- Modify common record due to exporter exhaustion. new common record type 10 adds 4 extra bytes. Reads v1 common record transparently +- Fix sflow potential crash 2013-11-13 v1.6.11 -- Add ASA/NSEL 9.x protcol changes -- Make it llvm compilable - + +- Add ASA/NSEL 9.x protcol changes +- Make it llvm compilable + 2013-08-12 v1.6.10p1 -- Fix -t +/- n timeslot option -- Fix bug in nfanon - stat record update. -- Fix bug in netflow v5 mudule: extension map size wrong. -- Fix bug nfexport: In some cases could result in wrong flow counter. -- Fix nftrack - could coredump in some cases. + +- Fix -t +/- n timeslot option +- Fix bug in nfanon - stat record update. +- Fix bug in netflow v5 mudule: extension map size wrong. +- Fix bug nfexport: In some cases could result in wrong flow counter. +- Fix nftrack - could coredump in some cases. 2013-05-16 v1.6.10 -- Fix SPARC compile/optimise bug -- Add output packet/bytes counter to global stat - importatnt for NSEL flows ASA > 8.5 -- Add NSEL filter options xnet -- Modify extension descriptor code for nfdump1.7. Still use 1.6 extension map layout for compatibility -- Add prototype for nfpcapd - pcap -> nfdump collector. Converts traffoc directly to nfdump files. -- Fix bug in ipfix module: uninitialised variable -- Cleanup syslog/LogError calls -- Fix minor non critical bugs and compile issues + +- Fix SPARC compile/optimise bug +- Add output packet/bytes counter to global stat - importatnt for NSEL flows ASA > 8.5 +- Add NSEL filter options xnet +- Modify extension descriptor code for nfdump1.7. Still use 1.6 extension map layout for compatibility +- Add prototype for nfpcapd - pcap -> nfdump collector. Converts traffoc directly to nfdump files. +- Fix bug in ipfix module: uninitialised variable +- Cleanup syslog/LogError calls +- Fix minor non critical bugs and compile issues 2013-03-02 v1.6.9 -- Fix some bugs in beta 1.6.9 NSEL code -- Fix bug statistics update with aggreagted flow records -- Fix sflow bug sfcapd stores wrong (ghost) dump by past samples in same sflow datagram + +- Fix some bugs in beta 1.6.9 NSEL code +- Fix bug statistics update with aggreagted flow records +- Fix sflow bug sfcapd stores wrong (ghost) dump by past samples in same sflow datagram 2012-12-31 -- Add time received in csv output -- ICMP should handled better now - somewhat -- Implement ASA NSEL records -- Add definitions in nffile and nx for ASA NSEL extensions + +- Add time received in csv output +- ICMP should handled better now - somewhat +- Implement ASA NSEL records +- Add definitions in nffile and nx for ASA NSEL extensions 2012-11-09 v1.6.8p1 -- Add dynamic source directory tree for multiple exporters -- Fix exporter bug: 'too many exporters' with large time windows -- Fix uninitialised exporter sysid in default sampler record - v9 -- Fix v9/ipfix cache initialisation with no templates > 1 in same packet + +- Add dynamic source directory tree for multiple exporters +- Fix exporter bug: 'too many exporters' with large time windows +- Fix uninitialised exporter sysid in default sampler record - v9 +- Fix v9/ipfix cache initialisation with no templates > 1 in same packet 2012-10-26 v1.6.8 -- Add ip list option for 'next ip' in filter syntax -- Accept v9 sampler_id in 2bytes -- Fix IPFIX mac address bug - did not get collected -- Add IPFIX packet/octet TotalCount fields 85/86 -- Add received timestamp to sflow collector -- Fix long flow duration calculation - 32bit overflow -- Fix v9 sampling ID: allow 2 byte ID -- Add IPFIX options as rfc5101 section-6.2 -- Add exporter records for sflow collector -- Fix bug for MAC address printing %idmc and %odmc. -- Add received time stamp extension -- Add recursive format parser. Allows to extend predefined formats. -- Change flow record sorting to heapsort. remove limit 1000 -- Merge -m option to -O tstart. -m now depricated. -- Add -O tend. Print order according to tend of flows ascending -- Apply -O print order for printing flow cache. Applies to -A + +- Add ip list option for 'next ip' in filter syntax +- Accept v9 sampler_id in 2bytes +- Fix IPFIX mac address bug - did not get collected +- Add IPFIX packet/octet TotalCount fields 85/86 +- Add received timestamp to sflow collector +- Fix long flow duration calculation - 32bit overflow +- Fix v9 sampling ID: allow 2 byte ID +- Add IPFIX options as rfc5101 section-6.2 +- Add exporter records for sflow collector +- Fix bug for MAC address printing %idmc and %odmc. +- Add received time stamp extension +- Add recursive format parser. Allows to extend predefined formats. +- Change flow record sorting to heapsort. remove limit 1000 +- Merge -m option to -O tstart. -m now depricated. +- Add -O tend. Print order according to tend of flows ascending +- Apply -O print order for printing flow cache. Applies to -A 2012-07-31 v1.6.7-tc-1 -- Special version for TC -- Print exporter and sampling records with nfdump -E -- Added exporter and sampling records to file. + +- Special version for TC +- Print exporter and sampling records with nfdump -E +- Added exporter and sampling records to file. 2012-07-30 v1.6.7 -- Prepare for file catalog in current file format. -- Fix bug in ReadBlock when reading flow from stdin pipe -- Add new more flexible translation engine for v9 -- Add nprobe client/server delay fields -- Prepare for NSEL merging -- Fix memory corruption with double -A flags -- Fix bug in nfreader with compat15 mode files + +- Prepare for file catalog in current file format. +- Fix bug in ReadBlock when reading flow from stdin pipe +- Add new more flexible translation engine for v9 +- Add nprobe client/server delay fields +- Prepare for NSEL merging +- Fix memory corruption with double -A flags +- Fix bug in nfreader with compat15 mode files 2012-03-12 v1.6.6 -- Minor IPFIX bug. -- IPFIX implement template withdraw -- For IPFIX, check packet sequence per template and observation domain -- Fix time window, when no flows collected or no flows matched - while processing -- Fixed typos -- Fix seg fault bug - test for EMPTY_LIST was missing at several places. + +- Minor IPFIX bug. +- IPFIX implement template withdraw +- For IPFIX, check packet sequence per template and observation domain +- Fix time window, when no flows collected or no flows matched while processing +- Fixed typos +- Fix seg fault bug - test for EMPTY_LIST was missing at several places. 2012-02-19 v1.6.6b1 -- Fix bps/pps. make it uint64_t, as bps/pps > 4Gb/s overflows. -- In record raw print mode: decode ICMP instead of src/dst ports -- sflow use announced exporter IP instead of sending IP for router ID -- sflow: Ignore extra fill bytes. Do not complain. -- sflow: fix packet length issue. -- Add IPFIX protokoll support + +- Fix bps/pps. make it uint64_t, as bps/pps > 4Gb/s overflows. +- In record raw print mode: decode ICMP instead of src/dst ports +- sflow use announced exporter IP instead of sending IP for router ID +- sflow: Ignore extra fill bytes. Do not complain. +- sflow: fix packet length issue. +- Add IPFIX protokoll support 2011-12-31 v1.6.5 -- Fix 64bit bug when using byte/packet limits -- for v5 and sampling use 64bit counters to prevent overflow for large sampled flows. -- Fixed Ident printig bug + +- Fix 64bit bug when using byte/packet limits +- for v5 and sampling use 64bit counters to prevent overflow for large sampled flows. +- Fixed Ident printig bug 2011-07-11 v1.6.4 -- some code restructuring - prepare for IPFIX module -- Add netflow v1 module. Some routers still use that -- Add %sn, %dn output tags for src/dst networks -- Fix buffer length check in v5. -- Fix export bug: include last flow cache bucket, when exporting -- number in all filter expressions accept hex values -- fix an sflow colletor bug. Missing extension maps in rotated files -- implement extended statistics. Currently ports and bpp distribution - vectors can be collected automatically be nfcapd. Still experimental + +- some code restructuring - prepare for IPFIX module +- Add netflow v1 module. Some routers still use that +- Add %sn, %dn output tags for src/dst networks +- Fix buffer length check in v5. +- Fix export bug: include last flow cache bucket, when exporting +- number in all filter expressions accept hex values +- fix an sflow colletor bug. Missing extension maps in rotated files +- implement extended statistics. Currently ports and bpp distribution vectors can be collected automatically be nfcapd. Still experimental 2011-02-26 v1.6.3p1 -- Fix timebug fix :(, make it a compile time option -- fix v7 sequence errors + +- Fix timebug fix :(, make it a compile time option +- fix v7 sequence errors 2011-02-15 -- Zero out unused fields after aggregation + +- Zero out unused fields after aggregation 2011-02-05 -- Fix SysUptime 32bit overflow in v5 header -- Add fix for strange first/last swap reported by some users. -2011-01-09 v1.6.3 -- Fix extension size bug -- Move IP anonymisation to separate binary nfanon -- Fix initialise bug of -o fmt: and not available fields +- Fix SysUptime 32bit overflow in v5 header +- Add fix for strange first/last swap reported by some users. + +2011-01-09 v1.6.3 + +- Fix extension size bug +- Move IP anonymisation to separate binary nfanon +- Fix initialise bug of -o fmt: and not available fields -2010-09-09 v1.6.2 -- released -- fixes some sflow bugs in sfcapd +2010-09-09 v1.6.2 + +- released +- fixes some sflow bugs in sfcapd 2010-04-28 v1.6.1p0 -- Update flow tools converter to build with Google-Code version 0.68.5 -- Fix sflow bugs + +- Update flow tools converter to build with Google-Code version 0.68.5 +- Fix sflow bugs 2010-03-05 v1.6.1 -- Fix bug in man page for -t -- Test sampler infos before using them ( nfcapd startup ) -- Add sampling tags #34, #35 used by JunOS -- nfexpire: Fix empty .nfsat, when setting limits on an empty directory -- Fix coredump for -B -m (-w) combination -- Optimise some extension map code + +- Fix bug in man page for -t +- Test sampler infos before using them ( nfcapd startup ) +- Add sampling tags #34, #35 used by JunOS +- nfexpire: Fix empty .nfsat, when setting limits on an empty directory +- Fix coredump for -B -m (-w) combination +- Optimise some extension map code 2009-12-28 stable v1.6 -- Few bug fixes in release candidates rc1, rc2 and rc3 + +- Few bug fixes in release candidates rc1, rc2 and rc3 2009-11-16 snapshot-1.6b-20091116 -- Update sflow collector with new tags -- Add router IP extension -- Add router ID (engine type/ID) extension + +- Update sflow collector with new tags +- Add router IP extension +- Add router ID (engine type/ID) extension 2009-09-30 snapshot-1.6b-20090930 -- snapshot bugfix release + +- snapshot bugfix release 2009-11-0801 snapshot-1.6b-20090806 -- Add srcmask and dstmask aggregation -- Add csv output mode. -o csv -- Fix some bugs of previous beta -- Add bidirectional aggregation of flows ( -b, -B ) -- Add possibility to save aggregated flows into file ( -w ) - Note: This results in a behaviour change for -w in combination - with aggragation ) -- Extend -N ( do not scale numbers ) to all text output not just summary -- Make extension handling more robust for some moody IOSes. -- Remove header lines of -s stat, when using -q ( quiet ) - Note: This results in a behaviour change for -N -- Remove -S option from nfdump ( legacy 1.4 compatibility ) -- Make use of log (syslog) functions for nfprofile. -- Move log functions to util.c + +- Add srcmask and dstmask aggregation +- Add csv output mode. -o csv +- Fix some bugs of previous beta +- Add bidirectional aggregation of flows ( -b, -B ) +- Add possibility to save aggregated flows into file ( -w ) Note: This results in a behaviour change for -w in combination with aggragation ) +- Extend -N ( do not scale numbers ) to all text output not just summary +- Make extension handling more robust for some moody IOSes. +- Remove header lines of -s stat, when using -q ( quiet ) Note: This results in a behaviour change for -N +- Remove -S option from nfdump ( legacy 1.4 compatibility ) +- Make use of log (syslog) functions for nfprofile. +- Move log functions to util.c 2009-06-19 snapshot-1.6b-20090717 -- Flow-tools converter updated - supports more common elements. -- Sflow collector updated. Supports more common elements. -- Add sampling to nfdump. Sampling is automatically recognised - in v5 undocumented header fields and in v9 option templates. - see nfcapd.1(1) -- Add @include option for filter to include more filter files. -- Add flexible aggregation comparable to Flexible Netflow (FNF) -- All new tags can be selected in -o fmt:... see nfdump(1) -- topN stat for all new tags is implemented -- Integrate developer code to read from pcap files into stable -- Update filter syntax for new tags -- Added more v9 tags for netflow v9. - The detailed tags are listed in nfcapd(1) - Adding new tags also extended the binary file format with - data block format 2, which is extension based. File format - for version <= 1.5.* ( Data block format 1 ) is read - transparently. Data block 2 are skipped by nfdump 1.5.7. - 32bit but AS and interface numbers are supported. -- Add flexible storage option for nfcapd. To save disk space, the - data extensions to be stored in the data file are user selectable. -- Added option for multiple netflow stream to same port. - -n - Example: -n router1,192.168.100.1,/var/nfdump/router1 - So multiple -n options may be given at the command line - Old style syntax still works for compatibility, ( -I .. -l ... ) - but then only one source is supported. -- Move to automake for building nfdump -- Switch scaling factor ( k, M, G ) from 1024 to 1000. -- Make nfdump fully 64bit compliant. ( 8bit data alignments and access ) + +- Flow-tools converter updated - supports more common elements. +- Sflow collector updated. Supports more common elements. +- Add sampling to nfdump. Sampling is automatically recognised in v5 undocumented header fields and in v9 option templates. see nfcapd.1(1) +- Add @include option for filter to include more filter files. +- Add flexible aggregation comparable to Flexible Netflow (FNF) +- All new tags can be selected in -o fmt:... see nfdump(1) +- topN stat for all new tags is implemented +- Integrate developer code to read from pcap files into stable +- Update filter syntax for new tags +- Added more v9 tags for netflow v9. The detailed tags are listed in nfcapd(1) Adding new tags also extended the binary file format with data block format 2, which is extension based. File format for version <= 1.5.\* ( Data block format 1 ) is read transparently. Data block 2 are skipped by nfdump 1.5.7. 32bit but AS and interface numbers are supported. +- Add flexible storage option for nfcapd. To save disk space, the data extensions to be stored in the data file are user selectable. +- Added option for multiple netflow stream to same port. -n Example: -n router1,192.168.100.1,/var/nfdump/router1 So multiple -n options may be given at the command line Old style syntax still works for compatibility, ( -I .. -l ... ) but then only one source is supported. +- Move to automake for building nfdump +- Switch scaling factor ( k, M, G ) from 1024 to 1000. +- Make nfdump fully 64bit compliant. ( 8bit data alignments and access ) 2009-04-17 stable 1.5.8 -- Fix daylight summer time bug, when guessing sub dirs. file access ( -M, -r ) -- Bug fixes for 64bits CPUs + +- Fix daylight summer time bug, when guessing sub dirs. file access ( -M, -r ) +- Bug fixes for 64bits CPUs 2008-02-22 stable-1,5.7 -- Add icmp type/code decoding -- Add proper icmp v9 decoding -- Fix memory leaks in -e auto expire mode in nfcapd. -- Fix somee potential dead locks with file locking, when expiring -- Fix multicast bug in nfreplay -- Add hostname lookup for IP addresses in filter. - -2007-10-15 stable-1.5.6 -- Fix odd CISCO behaviour for ICMP type/code in src port. -- Add fast LZO1X-1 compression option (-z) for output file. -- Add lists for port in syntax -> port in [ 135 137 445] -- Add lists for AS syntax -> as in [ 1024 1025 ] -- Bug fix in filter for syntax 'src as and dst as' - -2007-08-24 stable-1.5.5 -- Fix nfprofile bug, nfprofile crashes when last opts line is not valid for - some reason. -- Fix potential hand for nfexpire, on empty flow directories. - -2007-08-08 snapshot-20070808 -- Idents may contain '-' in name. -- Fixed install bugs in Makefile.in and configure.in -- Installs now cleanly on Solaris -- Handle 4byte interface numbers in v9. Quick fix: 4bytes reduced to 2bytes. -- Fix aggregation bug in statistics. -- ftok(3) C library call replaced by more reliable own implementation. - Did result in error messages like "Another collector is already running" -- Fix minor bugs iin file range selction -R. -- Add recursive behaviour for -R -- New option -i can canche Ident descriptor in data files. - -2007-03-12 snapshot-20070312 -- Bug fix release of 20070306 - -2007-03-06 snapshot-20070306 -- Fix bug in flist.c. Resulted in a coredump when using sub dirs and -R . ( all files ) -- Fix minor bug in nfcapd.c. -- Extend nfprofile for alerting system of nfsen - special version of profiles -- Extend nfprofile for shadow profiles. - -2007-08-10 snapshot-20070110 -- Fix some compiler warnings, when compiled on a 64bit LINUX -- Fixes an sflow bug: IP address was printed in wrong direction. ( lower bits first ) -- Add new IP addr taging option -T for easy parsing for nfsen lookups -- Add new IP list for massive address filtering: - syntax: ip in [ 12345 23456 3456 ....] -- Change nfprofile for channel based profiling. This breaks with old nfprofile - functionality. -- Remove space from ICMP type/code when followed by an IP address - -2006-07-21 snapshot-20060809 -- Make nfexpire ready for profile expiration -- Fix bug in nfrpofile. sub dir hierarchy not handled correctly. - -2006-07-21 snapshot-20060721 -- Add -N option for plain number output in summary line - -2006-07-21 snapshot-20060721 -- Do recursive file selection when a directory is given by -R - -2006-06-14 snapshot-20060621 -- Add srcas/dstas/proto aggregation. - Note: This changes the default aggregation behaviour, but gives more flexibility -- Add tos to element statistics list - -2006-06-14 snapshot-20060614 -- Add additional stat line at the end of output -- Add new binary nfexpire. Manages data expiry on time and/or size based limits - Includes new bookkeeping records in nfcapd. See nfexpire(1) -- Add ICMP type/code decoding in flow listing instead of dst port -- Add packet repeater in nfcapd/sfcapd. In addition, incoming UDP packets can - be directly forwarded to another IP address/Port. See new option -R -- Add sub directory hierarchies: Files can be stored into various sub dir levels - based on different time formats. see new option -S -- Some minor bug fixes. -- Code cleanup in nfcapd. better daemonize code and communication with launcher. - -2006-04-xx v.1.5.1 - Fix bug in nfdump.c: Writing anonymized flows to file did not work corretly - stdin input format now compatible with file format, therefore - 'nfdump < file' works again as it did in nfdump 1.4. - Fix bug in nfcapd.c: Error handling not correct when receiving a non - recognized netflow packet. Resulted in an endless loop -2006-03-27 snapshot 1.5-20060327 - Make all element statistics -s transport layer protocol - independant by default. Add :p to stat name ( e.g. srcip:p ) to - enable transport layer dependant statistics on request. -2006-03-20 snapshot 1.5-20060320 - Fix bug in filter engine: 'not flags xyz' produces wrong results - when more than a single flag is specified. - Minor man page fixes. - -2006-03-06 v1.5 - Fix bug nfcapd. Laucher signaled too early. File not yet properly - closed. -2006-02-14 v1.5-beta-5 - Add srcas, dstas, input and output interfaces in aggregated - output. - Fix IPv6 bug in filter: accept 1234:: address. - rename nfcapd.curent tmp file to nfcapd.curren.. Poorly - configured nfcapd processes may mess up themselves otherwise. -2006-02-02 v1.5-beta-4 - Fix netflow v5 dPkts <-> dOctets collector bug. - Update pipe format to include more information - Allow AS number 0 in filter syntax. - Add some more boundary checking - netflow exporters aren't bug free either - sigh .. -2006-01-11 v1.5-beta-3 - Fix isnumber incompatibility in grammar.y - Add 'if' statistics -2006-01-10 v1.5-beta-2 -nf_common.c Fix bug in format parser. - Extended 'proto ' syntax to support all protocols - Change time format in summary line to ISO format -2005-12-20 v1.5-beta-1 -*.* A lot of internal changes, not mentioned here. :( - -nfdump Add subnet aggregation for option -A - A new syntax e.g. srcip4/24, dstip6/64 is supported for subnet wise aggregation. - example: traffic of a whole subnet -A srcip4/24 -s srcip/bytes - -nfdump Add more stat element option. -s now supports: - srcip, dstip, ip, srcport, dstport, port, srcas, dstas, as, inif, outif, proto - -nfdump Add -z. Suppress writing flows to data files. Only stat information is written. -nfprofile Used only be nfsen for upcoming shadow profiles. If you don't understand this - simply ignore it. - -nfdump Add -q option to suppress header as well as stat information at the bottom -nfprofile for easier post processing with external programms. - -nf_common.c Output format processsing rewritting for more flexibility. Besides standard -nfdump.c output formats line, long extended etc., user defined output formats are now - possible and can even be compiled into nfdump for easy access. See -o fmt: - and nfdump.c around line 100. - -*.* Integrate netflow v9 into nfdump. Only a subset of v9 is stored into - the data files, basically everything needed for nfdump to work as it did before. - This also includes IPv6 support for any nfdump options. CryptoPAN extended - to work with IPv6. IPv6 condensed output format for better readability. - Output formats available in long and condensed mode: e.g. line -> line6 - extended -> extended6 - -*.* Replace binary data file format. Old format not flexible enough for - upcoming netflow v9/sflow data. *.stat files are gone. The same - information is now available under nfdump -I - New format about 5% larger in size, but faster for reading and writing. - speed gain eaten up by more complex processing - sigh .. - compat14 mode enables transparent reading of old style format. - nffile.[ch] now handles all data file stuff. - -nfreplay Multicast enabled: - Add -j . Joins the specified multicast group ( v4 or v6 ) - sending flows to this group. - -nfreplay IPv6 enabled: - Add option -4 and -6 to force a specific protocol, otherwise - protocol is automatically selected according the hostname to send flows to. - Add -K key, to send data anonymized, using CryptoPAn - -nfcapd Multicast enabled: - Add -j . Joins the specified multicast group ( v4 or v6 ) - for listening. - -nfcapd IPv6 enabled: - Add option -4 and -6 for IPv4 and IPv6. By default, listen on IPv4. - Option -b to bind for a specific host/IP address automatically - selects appropriate protocol. - -nfnet.c All functions to setup network sockets for listening/sending are - put into this file. + +- Add icmp type/code decoding +- Add proper icmp v9 decoding +- Fix memory leaks in -e auto expire mode in nfcapd. +- Fix somee potential dead locks with file locking, when expiring +- Fix multicast bug in nfreplay +- Add hostname lookup for IP addresses in filter. + +2007-10-15 stable-1.5.6 + +- Fix odd CISCO behaviour for ICMP type/code in src port. +- Add fast LZO1X-1 compression option (-z) for output file. +- Add lists for port in syntax -> port in [ 135 137 445] +- Add lists for AS syntax -> as in [ 1024 1025 ] +- Bug fix in filter for syntax 'src as and dst as' + +2007-08-24 stable-1.5.5 + +- Fix nfprofile bug, nfprofile crashes when last opts line is not valid for some reason. +- Fix potential hand for nfexpire, on empty flow directories. + +2007-08-08 snapshot-20070808 + +- Idents may contain '-' in name. +- Fixed install bugs in Makefile.in and configure.in +- Installs now cleanly on Solaris +- Handle 4byte interface numbers in v9. Quick fix: 4bytes reduced to 2bytes. +- Fix aggregation bug in statistics. +- ftok(3) C library call replaced by more reliable own implementation. Did result in error messages like "Another collector is already running" +- Fix minor bugs iin file range selction -R. +- Add recursive behaviour for -R +- New option -i can canche Ident descriptor in data files. + +2007-03-12 snapshot-20070312 + +- Bug fix release of 20070306 + +2007-03-06 snapshot-20070306 + +- Fix bug in flist.c. Resulted in a coredump when using sub dirs and -R . ( all files ) +- Fix minor bug in nfcapd.c. +- Extend nfprofile for alerting system of nfsen - special version of profiles +- Extend nfprofile for shadow profiles. + +2007-08-10 snapshot-20070110 + +- Fix some compiler warnings, when compiled on a 64bit LINUX +- Fixes an sflow bug: IP address was printed in wrong direction. ( lower bits first ) +- Add new IP addr taging option -T for easy parsing for nfsen lookups +- Add new IP list for massive address filtering: syntax: ip in [ 12345 23456 3456 ....] +- Change nfprofile for channel based profiling. This breaks with old nfprofile functionality. +- Remove space from ICMP type/code when followed by an IP address + +2006-07-21 snapshot-20060809 + +- Make nfexpire ready for profile expiration +- Fix bug in nfrpofile. sub dir hierarchy not handled correctly. + +2006-07-21 snapshot-20060721 + +- Add -N option for plain number output in summary line + +2006-07-21 snapshot-20060721 + +- Do recursive file selection when a directory is given by -R + +2006-06-14 snapshot-20060621 + +- Add srcas/dstas/proto aggregation. Note: This changes the default aggregation behaviour, but gives more flexibility +- Add tos to element statistics list + +2006-06-14 snapshot-20060614 + +- Add additional stat line at the end of output +- Add new binary nfexpire. Manages data expiry on time and/or size based limits Includes new bookkeeping records in nfcapd. See nfexpire(1) +- Add ICMP type/code decoding in flow listing instead of dst port +- Add packet repeater in nfcapd/sfcapd. In addition, incoming UDP packets can be directly forwarded to another IP address/Port. See new option -R +- Add sub directory hierarchies: Files can be stored into various sub dir levels based on different time formats. see new option -S +- Some minor bug fixes. +- Code cleanup in nfcapd. better daemonize code and communication with launcher. + +2006-04-xx v.1.5.1 Fix bug in nfdump.c: Writing anonymized flows to file did not work corretly stdin input format now compatible with file format, therefore 'nfdump < file' works again as it did in nfdump 1.4. Fix bug in nfcapd.c: Error handling not correct when receiving a non recognized netflow packet. Resulted in an endless loop 2006-03-27 snapshot 1.5-20060327 Make all element statistics -s transport layer protocol independant by default. Add :p to stat name ( e.g. srcip:p ) to enable transport layer dependant statistics on request. 2006-03-20 snapshot 1.5-20060320 Fix bug in filter engine: 'not flags xyz' produces wrong results when more than a single flag is specified. Minor man page fixes. + +2006-03-06 v1.5 Fix bug nfcapd. Laucher signaled too early. File not yet properly closed. 2006-02-14 v1.5-beta-5 Add srcas, dstas, input and output interfaces in aggregated output. Fix IPv6 bug in filter: accept 1234:: address. rename nfcapd.curent tmp file to nfcapd.curren.. Poorly configured nfcapd processes may mess up themselves otherwise. 2006-02-02 v1.5-beta-4 Fix netflow v5 dPkts <-> dOctets collector bug. Update pipe format to include more information Allow AS number 0 in filter syntax. Add some more boundary checking - netflow exporters aren't bug free either - sigh .. 2006-01-11 v1.5-beta-3 Fix isnumber incompatibility in grammar.y Add 'if' statistics 2006-01-10 v1.5-beta-2 nf_common.c Fix bug in format parser. Extended 'proto ' syntax to support all protocols Change time format in summary line to ISO format 2005-12-20 v1.5-beta-1 _._ A lot of internal changes, not mentioned here. :( + +nfdump Add subnet aggregation for option -A A new syntax e.g. srcip4/24, dstip6/64 is supported for subnet wise aggregation. example: traffic of a whole subnet -A srcip4/24 -s srcip/bytes + +nfdump Add more stat element option. -s now supports: srcip, dstip, ip, srcport, dstport, port, srcas, dstas, as, inif, outif, proto + +nfdump Add -z. Suppress writing flows to data files. Only stat information is written. nfprofile Used only be nfsen for upcoming shadow profiles. If you don't understand this simply ignore it. + +nfdump Add -q option to suppress header as well as stat information at the bottom nfprofile for easier post processing with external programms. + +nf_common.c Output format processsing rewritting for more flexibility. Besides standard nfdump.c output formats line, long extended etc., user defined output formats are now possible and can even be compiled into nfdump for easy access. See -o fmt: and nfdump.c around line 100. + +_._ Integrate netflow v9 into nfdump. Only a subset of v9 is stored into the data files, basically everything needed for nfdump to work as it did before. This also includes IPv6 support for any nfdump options. CryptoPAN extended to work with IPv6. IPv6 condensed output format for better readability. Output formats available in long and condensed mode: e.g. line -> line6 extended -> extended6 + +_._ Replace binary data file format. Old format not flexible enough for upcoming netflow v9/sflow data. \*.stat files are gone. The same information is now available under nfdump -I New format about 5% larger in size, but faster for reading and writing. speed gain eaten up by more complex processing - sigh .. compat14 mode enables transparent reading of old style format. nffile.[ch] now handles all data file stuff. + +nfreplay Multicast enabled: Add -j . Joins the specified multicast group ( v4 or v6 ) sending flows to this group. + +nfreplay IPv6 enabled: Add option -4 and -6 to force a specific protocol, otherwise protocol is automatically selected according the hostname to send flows to. Add -K key, to send data anonymized, using CryptoPAn + +nfcapd Multicast enabled: Add -j . Joins the specified multicast group ( v4 or v6 ) for listening. + +nfcapd IPv6 enabled: Add option -4 and -6 for IPv4 and IPv6. By default, listen on IPv4. Option -b to bind for a specific host/IP address automatically selects appropriate protocol. + +nfnet.c All functions to setup network sockets for listening/sending are put into this file. 2005-08-22 v1.4 -- nfreplay: Bug fix sending flows. -- nfdump: Add CryptoPAn code to anonymize IP addresses. New option -K -- nfdump: Change time format in output to ISO 8601 compatible: e.g. 1981-04-05 14:30:30.100 -- nfdump: Add scaling factor k,m,g to number in filter syntax: e.g. bytes > 1m -- nfdump: Create new output format extended with additional fields pps, bps and bpp -- nfdump: Rename output format extended to raw -- nfdump: More than one single flow element statistic ( -s ) is now possible -- nfdump: Add user defined sort order in flow element statistic -- nfdump: Flow element statistic can be ordered by more than one order in the same run -- nfdump: Add pps, bps and bpp fields in flow element statistics -- nfdump: Add more symbolic protocols ESP, AH, GRP and RVSP to filter syntax -- nfdump: Add duration, pps, bps and bpp to filter syntax -- nfdump: Make nfdump miliseconds aware. Older versions skipped msecs. - Binary nfdump file format changed due to this. - output formats changed, due to this. -- nfdump: Add interface in/out if syntax to filter -- nfcapd: Add flow_sequence check. Reports missing flows now. -- nfcapd: Report statistics to syslog LOG_INFO when data file is rotated. -- ft2nfdump: Add ft2nfdump to read netflow data from flow-tools + +- nfreplay: Bug fix sending flows. +- nfdump: Add CryptoPAn code to anonymize IP addresses. New option -K +- nfdump: Change time format in output to ISO 8601 compatible: e.g. 1981-04-05 14:30:30.100 +- nfdump: Add scaling factor k,m,g to number in filter syntax: e.g. bytes > 1m +- nfdump: Create new output format extended with additional fields pps, bps and bpp +- nfdump: Rename output format extended to raw +- nfdump: More than one single flow element statistic ( -s ) is now possible +- nfdump: Add user defined sort order in flow element statistic +- nfdump: Flow element statistic can be ordered by more than one order in the same run +- nfdump: Add pps, bps and bpp fields in flow element statistics +- nfdump: Add more symbolic protocols ESP, AH, GRP and RVSP to filter syntax +- nfdump: Add duration, pps, bps and bpp to filter syntax +- nfdump: Make nfdump miliseconds aware. Older versions skipped msecs. Binary nfdump file format changed due to this. output formats changed, due to this. +- nfdump: Add interface in/out if syntax to filter +- nfcapd: Add flow_sequence check. Reports missing flows now. +- nfcapd: Report statistics to syslog LOG_INFO when data file is rotated. +- ft2nfdump: Add ft2nfdump to read netflow data from flow-tools 2005-04-21 v1.3 -- Add option -A for more flexible aggregation. -- Correct spelling errors :( - -2005-03-04 v1.2.1 -Bug fix release -- nfcapd: launcher subprocess may hang on Linux 2.6.x kernels. - Cleaned up interrupt handling. -- nfcapd: fix include order of socket.h and types.h in order to - compile cleanly under FreeBSD 4.x -- nfcapd: clean up syslog logging. -- nfdump: Multiple sources ( -M ) and sort flows ( -m ) with - -c did not list the correct flows. -- nfprofile: Profiling with multiple sources may produce incorrect - profiles. + +- Add option -A for more flexible aggregation. +- Correct spelling errors :( + +2005-03-04 v1.2.1 Bug fix release + +- nfcapd: launcher subprocess may hang on Linux 2.6.x kernels. Cleaned up interrupt handling. +- nfcapd: fix include order of socket.h and types.h in order to compile cleanly under FreeBSD 4.x +- nfcapd: clean up syslog logging. +- nfdump: Multiple sources ( -M ) and sort flows ( -m ) with -c did not list the correct flows. +- nfprofile: Profiling with multiple sources may produce incorrect profiles. 2004-12-20 v1.2 -- nfcapd handles transparent v5 and v7 flows. v7 gets converted into v5 -- nfcapd can execute any command at the end of interval. New option -x -- nfdump Extended filter syntax for flags, to, bytes and packets -- Rearrange output formats in nfdump: new switch -o, remove switch -E - output formats: 'line', 'long', 'extended' and 'pipe' -- More flexible statistic handling in nfdump: cleanup ugly -s -s -s - syntax. Replaced by -s option. New statistics for Port and AS. - -2004-09-20 v 1.1 -First public Version. + +- nfcapd handles transparent v5 and v7 flows. v7 gets converted into v5 +- nfcapd can execute any command at the end of interval. New option -x +- nfdump Extended filter syntax for flags, to, bytes and packets +- Rearrange output formats in nfdump: new switch -o, remove switch -E output formats: 'line', 'long', 'extended' and 'pipe' +- More flexible statistic handling in nfdump: cleanup ugly -s -s -s syntax. Replaced by -s option. New statistics for Port and AS. + +2004-09-20 v 1.1 First public Version. diff --git a/README.md b/README.md index c747fdb3..5a3c2918 100755 --- a/README.md +++ b/README.md @@ -1,152 +1,118 @@ # nfdump -Stable Release v1.6.23 +Stable Release v1.6.24 -See the Changelog file for all changes in release 1.6.23 +See the Changelog file for all changes in release 1.6.24 -nfdump is a toolset in order to collect and process netflow and sflow data, sent from netflow/sflow compatible devices. -The toolset supports netflow __v1__, __v5/v7__,__v9__,__IPFIX__ and __SFLOW__. nfdump supports IPv4 as well as IPv6. +nfdump is a toolset in order to collect and process netflow and sflow data, sent from netflow/sflow compatible devices. The toolset supports netflow **v1**, **v5/v7**,**v9**,**IPFIX** and **SFLOW**. nfdump supports IPv4 as well as IPv6. -__Note:__ nfdump 1.6.18 and newer versions __not longer__ support nfdump-1.5.x files. If you have nfdump-1.5.x please convert them -before upgrading. +**Note:** nfdump 1.6.18 and newer versions **not longer** support nfdump-1.5.x files. If you have nfdump-1.5.x please convert them before upgrading. -nfdump is used as backend toolset for __NfSen__. +nfdump is used as backend toolset for **NfSen**. --- ## NSEL/ASA, NEL/NAT support -__NSEL__ (Network Event Security Logging) as well as NEL (NAT Event Logging) are technologies invented by __CISCO__ and also use the netflow v9 protocol. However, NSEL and NEL are not flows as commonly known but rather *__Events__!* exported from specific devices such as CISCO ASA. nfdump supports Event looging as part of netflow v9. +**NSEL** (Network Event Security Logging) as well as NEL (NAT Event Logging) are technologies invented by **CISCO** and also use the netflow v9 protocol. However, NSEL and NEL are not flows as commonly known but rather _**Events**!_ exported from specific devices such as CISCO ASA. nfdump supports Event looging as part of netflow v9. -__Note:__ The older nfdump-1.5.8-2-NSEL is __not compatible__ with nfdump > 1.6.9 which supports NSEL/NEL. +**Note:** The older nfdump-1.5.8-2-NSEL is **not compatible** with nfdump > 1.6.9 which supports NSEL/NEL. -__Junos NAT Event Logging__ is mostly compatible with CISCO's NAT Event Logging - mostly - it needs another data interpretation. -See __--enable-jnat__ below +**Junos NAT Event Logging** is mostly compatible with CISCO's NAT Event Logging - mostly - it needs another data interpretation. See **--enable-jnat** below --- ## IPFIX -nfdump contains an IPFIX module for decoding IPFIX flow data. It -does not support the full IPFIX definition. +nfdump contains an IPFIX module for decoding IPFIX flow data. It does not support the full IPFIX definition. -* Supports basically same feature set of elements as netflow_v9 module -* Only UDP traffic is accepted no TCP/SCTP -* If you would like to see more IPFIX support, please contact me. +- Supports basically same feature set of elements as netflow_v9 module +- Only UDP traffic is accepted no TCP/SCTP +- If you would like to see more IPFIX support, please contact me. --- - ## Overview ### Building and config options -The toolset is build upon the autotools framework. Run `./autogen.sh` first. -Afterwards `./configure` `make` and `make install` should do the trick. +The toolset is build upon the autotools framework. Run `./autogen.sh` first. Afterwards `./configure` `make` and `make install` should do the trick. The following config options are available: -* __--enable-nsel__ -Compile nfdump, to read and process NSEL/NEL event data; default is __NO__ -* __--enable-jnat__ -compile nfdump, to read and process JunOS NAT event logging __NO__ -* __--enable-ftconv__ -Build the flow-tools to nfdump converter; default is __NO__ -* __--enable-sflow__ -Build sflow collector sfcpad; default is __NO__ -* __--enable-nfprofile__ -Build nfprofile used by NfSen; default is __NO__ -* __--enable-nftrack__ -Build nftrack used by PortTracker; default is __NO__ - -This code no longer reads nfdump-1.5.x data files. If needed use nfdump up -to v1.6.17 +- **--enable-nsel** + Compile nfdump, to read and process NSEL/NEL event data; default is **NO** +- **--enable-jnat** + compile nfdump, to read and process JunOS NAT event logging **NO** +- **--enable-ftconv** + Build the flow-tools to nfdump converter; default is **NO** +- **--enable-sflow** + Build sflow collector sfcpad; default is **NO** +- **--enable-nfprofile** + Build nfprofile used by NfSen; default is **NO** +- **--enable-nftrack** + Build nftrack used by PortTracker; default is **NO** + +This code no longer reads nfdump-1.5.x data files. If needed use nfdump up to v1.6.17 Development and beta options -* __--enable-devel__ -Insert lots of debug and development code into nfdump for testing and debugging; default is __NO__ -* __--enable-readpcap__ -Add code to nfcapd to read flow data also from pcap files; default is __NO__ -* __--enable-nfpcapd__ -Build nfpcapd collector to create netflow data from interface traffic or precollected pcap traffic, similar to softflowd; default is __NO__ - +- **--enable-devel** + Insert lots of debug and development code into nfdump for testing and debugging; default is **NO** +- **--enable-readpcap** + Add code to nfcapd to read flow data also from pcap files; default is **NO** +- **--enable-nfpcapd** + Build nfpcapd collector to create netflow data from interface traffic or precollected pcap traffic, similar to softflowd; default is **NO** ### The tools -__nfcapd__ - netflow collector daemon. -Collects the netflow data, sent from exporters and stores the flow records -into files. Automatically rotates files every n minutes. ( typically -every 5 min ) The netflow versions mentioned above are read transparently -Multiple netflow streams can be collected by a single or collector. + +**nfcapd** - netflow collector daemon. +Collects the netflow data, sent from exporters and stores the flow records into files. Automatically rotates files every n minutes. ( typically every 5 min ) The netflow versions mentioned above are read transparently Multiple netflow streams can be collected by a single or collector. nfcapd can listen on IPv6 or IPv4. Furthermore multicast is supported. -__nfdump__ - process collected netflow records. -Nfdump reads the netflow data from one or many files stored by nfcapd. -It's filter syntax is similar to tcpdump ( pcap like ) but adapted for netflow. -If you like tcpdump you will like nfdump. nfdump displays netflow -data and/or creates top N statistics of flows, bytes, packets. nfdump -has a powerful and flexible flow aggregation including bi-directional -flows. The output format is user selectable and also includes a simple -csv format for post processing. +**nfdump** - process collected netflow records. +Nfdump reads the netflow data from one or many files stored by nfcapd. It's filter syntax is similar to tcpdump ( pcap like ) but adapted for netflow. If you like tcpdump you will like nfdump. nfdump displays netflow data and/or creates top N statistics of flows, bytes, packets. nfdump has a powerful and flexible flow aggregation including bi-directional flows. The output format is user selectable and also includes a simple csv format for post processing. -__nfanon__ - anonymize netflow records +**nfanon** - anonymize netflow records IP addresses in flow records are anonimized using the CryptoPAn method. -__nfexpire__ - expire old netflow data +**nfexpire** - expire old netflow data Manages data expiration. Sets appropriate limits. Used by NfSen. -__nfreplay__ - netflow replay -Reads the netflow data from the files stored by nfcapd and sends it -over the network to another host. +**nfreplay** - netflow replay +Reads the netflow data from the files stored by nfcapd and sends it over the network to another host. #### Optional binaries: -__nfpcapd__ - pcap to netflow collector daemon -nfpcapd listens on a network interface, or reads precollected pcap traffic -and stores flow records into nfcapd comaptible files. It is nfcapd's -companion to convert traffic directly into nfdump records. +**nfpcapd** - pcap to netflow collector daemon +nfpcapd listens on a network interface, or reads precollected pcap traffic and stores flow records into nfcapd comaptible files. It is nfcapd's companion to convert traffic directly into nfdump records. -__sfcapd__ - sflow collector daemon -scfapd collects sflow data and stores it into nfcapd comaptible files. -"sfcapd includes sFlow(TM) code, freely available from https://github.com/sflow/sflowtool. +**sfcapd** - sflow collector daemon +scfapd collects sflow data and stores it into nfcapd comaptible files. "sfcapd includes sFlow(TM) code, freely available from https://github.com/sflow/sflowtool. -__nfprofile__ - netflow profiler. Required by NfSen -Reads the netflow data from the files stored by nfcapd. Filters the -netflow data according to the specified filter sets ( profiles ) and -stores the filtered data into files for later use. +**nfprofile** - netflow profiler. Required by NfSen +Reads the netflow data from the files stored by nfcapd. Filters the netflow data according to the specified filter sets ( profiles ) and stores the filtered data into files for later use. -__nftrack__ - Port tracking decoder for NfSen plugin PortTracker. +**nftrack** - Port tracking decoder for NfSen plugin PortTracker. -__ft2nfdump__ - flow-tools flow converter -ft2nfdump converts flow-tools data into nfdump format. +**ft2nfdump** - flow-tools flow converter +ft2nfdump converts flow-tools data into nfdump format. -__nfreader__ - Framework for programmers -nfreader is a framework to read nfdump files for any other purpose. -Own C code can be added to process flows. nfreader is not installed +**nfreader** - Framework for programmers +nfreader is a framework to read nfdump files for any other purpose. Own C code can be added to process flows. nfreader is not installed -__parse_csv.pl__ - Simple reader, written in Perl. -parse_csv.pl reads nfdump csv output and print the flows to stdout. -This program is intended to be a framework for post processing flows -for any other purpose. +**parse_csv.pl** - Simple reader, written in Perl. +parse_csv.pl reads nfdump csv output and print the flows to stdout. This program is intended to be a framework for post processing flows for any other purpose. #### Notes for sflow users: -sfcapd and nfcapd can be used concurrently to collect netflow and sflow -data at the same time. Generic command line options apply to both -collectors likewise. sfcapd's sflow decoding module is based on InMon's -sflowtool code and supports similar fields as nfcapd does for netflow v9, -which is a subset of all available sflow fields in an sflow record. -More fields may be integrated in future versions of sfcapd. + +sfcapd and nfcapd can be used concurrently to collect netflow and sflow data at the same time. Generic command line options apply to both collectors likewise. sfcapd's sflow decoding module is based on InMon's sflowtool code and supports similar fields as nfcapd does for netflow v9, which is a subset of all available sflow fields in an sflow record. More fields may be integrated in future versions of sfcapd. --- ### Compression -Binary data files can optionally be compressed using either the fast LZO1X-1 compression, -LZ4 or the efficient but slow bzip2 method. -If you compress automatically flows while they are collected, LZO1X-1 or LZ4 methods are -recommended. bzip2 uses about 30 times more CPU than LZO1X-1. Use bzip2 to archive netflow -data, which may reduce the disk usage again by a factor of 2. The compression of flow files -can be changed any time with nfdump -J -For more details on each methde, see: + +Binary data files can optionally be compressed using either the fast LZO1X-1 compression, LZ4 or the efficient but slow bzip2 method. If you compress automatically flows while they are collected, LZO1X-1 or LZ4 methods are recommended. bzip2 uses about 30 times more CPU than LZO1X-1. Use bzip2 to archive netflow data, which may reduce the disk usage again by a factor of 2. The compression of flow files can be changed any time with nfdump -J For more details on each methde, see: LZO1X-1: http://www.oberhumer.com/opensource/lzo @@ -154,71 +120,51 @@ LZ4: https://github.com/lz4/lz4 bzip2: http://www.bzip.org -You can check the compression speed for your system by running ./nftest . +You can check the compression speed for your system by running ./nftest . --- ## General Operation of nfdump -The goal of the design is to able to analyze netflow data from -the past as well as to track interesting traffic patterns -continuously. The amount of time back in the past is limited only -by the disk storage available for all the netflow data. The tools -are optimized for speed for efficient filtering. The filter rules -should look familiar to the syntax of tcpdump ( pcap compatible ). -All data is stored to disk, before it gets analyzed. This separates -the process of storing and analyzing the data. +The goal of the design is to able to analyze netflow data from the past as well as to track interesting traffic patterns continuously. The amount of time back in the past is limited only by the disk storage available for all the netflow data. The tools are optimized for speed for efficient filtering. The filter rules should look familiar to the syntax of tcpdump ( pcap compatible ). + +All data is stored to disk, before it gets analyzed. This separates the process of storing and analyzing the data. The data is organized in a time-based fashion. Every n minutes -- typically 5 min - nfcapd rotates and renames the output file -with the timestamp nfcapd.YYYYMMddhhmm of the interval e.g. -nfcapd.200907110845 contains data from July 11th 2009 08:45 onward. -Based on a 5min time interval, this results in 288 files per day. -Analyzing the data can be done for a single file, or by concatenating -several files for a single output. The output is either ASCII text -or binary data, when saved into a file, ready to be processed again -with the same tools. +- typically 5 min - nfcapd rotates and renames the output file with the timestamp nfcapd.YYYYMMddhhmm of the interval e.g. nfcapd.200907110845 contains data from July 11th 2009 08:45 onward. Based on a 5min time interval, this results in 288 files per day. + +Analyzing the data can be done for a single file, or by concatenating several files for a single output. The output is either ASCII text or binary data, when saved into a file, ready to be processed again with the same tools. -You may have several netflow sources - let's say 'router1' 'router2' -and so on. The data is organized as follows: +You may have several netflow sources - let's say 'router1' 'router2' and so on. The data is organized as follows: - /flow_base_dir/router1 - /flow_base_dir/router2 + /flow_base_dir/router1 + /flow_base_dir/router2 which means router1 and router2 are subdirs of the flow_base_dir. -Although several flow sources can be sent to a single collector, -It's recommended to have multiple collector on busy networks for -each source. -Example: Start two collectors on different ports: +Although several flow sources can be sent to a single collector, It's recommended to have multiple collector on busy networks for each source. Example: Start two collectors on different ports: - nfcapd -w -D -S 2 -B 1024000 -l /flow_base_dir/router1 -p 23456 - nfcapd -w -D -S 2 -B 1024000 -l /flow_base_dir/router2 -p 23457 + nfcapd -w -D -S 2 -B 1024000 -l /flow_base_dir/router1 -p 23456 + nfcapd -w -D -S 2 -B 1024000 -l /flow_base_dir/router2 -p 23457 -nfcapd can handle multiple flow sources. -All sources can go into a single file or can be split: +nfcapd can handle multiple flow sources. All sources can go into a single file or can be split: All into the same file: - nfcapd -w -D -S 2 -l /flow_base_dir/routers -p 23456 + nfcapd -w -D -S 2 -l /flow_base_dir/routers -p 23456 Collected on one port and split per source: - nfcapd -w -D -S 2 -n router1,172.16.17.18,/flow_base_dir/router1 \-n router2,172.16.17.20,/flow_base_dir/router2 -p 23456 + nfcapd -w -D -S 2 -n router1,172.16.17.18,/flow_base_dir/router1 \-n router2,172.16.17.20,/flow_base_dir/router2 -p 23456 See nfcapd(1) for a detailed explanation of all options. -Security: none of the tools requires root privileges, unless you have -a port < 1024. However, there is no access control mechanism in nfcapd. -It is assumed, that host level security is in place to filter the -proper IP addresses. +Security: none of the tools requires root privileges, unless you have a port < 1024. However, there is no access control mechanism in nfcapd. It is assumed, that host level security is in place to filter the proper IP addresses. -See the manual pages or use the -h switch for details on using each of -the programs. For any questions send email to peter@people.ops-trust.net +See the manual pages or use the -h switch for details on using each of the programs. For any questions send email to peter@people.ops-trust.net -Configure your router to export netflow. See the relevant documentation -for your model. +Configure your router to export netflow. See the relevant documentation for your model. A generic Cisco sample configuration enabling NetFlow on an interface: @@ -226,17 +172,14 @@ A generic Cisco sample configuration enabling NetFlow on an interface: interface fastethernet 0/0 ip route-cache flow -To tell the router where to send the NetFlow data, enter the following -global configuration command: +To tell the router where to send the NetFlow data, enter the following global configuration command: - ip flow-export 192.168.92.218 9995 - ip flow-export version 5 - - ip flow-cache timeout active 5 + ip flow-export 192.168.92.218 9995 + ip flow-export version 5 -This breaks up long-lived flows into 5-minute segments. You can choose -any number of minutes between 1 and 60; + ip flow-cache timeout active 5 +This breaks up long-lived flows into 5-minute segments. You can choose any number of minutes between 1 and 60; Netflow v9 full export example of a cisco 7200 with sampling enabled: @@ -265,158 +208,138 @@ Netflow v9 full export example of a cisco 7200 with sampling enabled: ip flow-export template timeout-rate 1 ip flow-export destination 192.168.92.218 9995 - See the relevant documentation for a full description of netflow commands -Note: Netflow version v5 and v7 have 32 bit counter values. The number of -packets or bytes may overflow this value, within the flow-cache timeout -on very busy routers. To prevent overflow, you may consider to reduce the -flow-cache timeout to lower values. All nfdump tools use 64 bit counters -internally, which means, all aggregated values are correctly reported. +Note: Netflow version v5 and v7 have 32 bit counter values. The number of packets or bytes may overflow this value, within the flow-cache timeout on very busy routers. To prevent overflow, you may consider to reduce the flow-cache timeout to lower values. All nfdump tools use 64 bit counters internally, which means, all aggregated values are correctly reported. -The binary format of the data files is netflow version independent. -For speed reasons the binary format is machine architecture dependent, and -as such can not be exchanged between little and big endian systems. -Internally nfdump does all processing IP protocol independent, which means -everything works for IPv4 as well as IPv6 addresses. -See the nfdump(1) man page for details. +The binary format of the data files is netflow version independent. For speed reasons the binary format is machine architecture dependent, and as such can not be exchanged between little and big endian systems. Internally nfdump does all processing IP protocol independent, which means everything works for IPv4 as well as IPv6 addresses. See the nfdump(1) man page for details. -netflow version 9: -nfcapd supports a large range of netflow v9 tags. Version 1.6 nfdump -supports the following fields. This list can be found in netflow_v9.h +netflow version 9: nfcapd supports a large range of netflow v9 tags. Version 1.6 nfdump supports the following fields. This list can be found in netflow_v9.h --- ### Flowset record types -Tag | ID -----|--- -NF9_IN_BYTES | 1 -IN_PACKETS | 2 -NF9_FLOWS_AGGR | 3 -NF9_IN_PROTOCOL | 4 -NF9_SRC_TOS | 5 -NF9_TCP_FLAGS | 6 -NF9_L4_SRC_PORT | 7 -NF9_IPV4_SRC_ADDR | 8 -NF9_SRC_MASK | 9 -NF9_INPUT_SNMP | 10 -NF9_L4_DST_PORT | 11 -NF9_IPV4_DST_ADDR | 12 -NF9_DST_MASK | 13 -NF9_OUTPUT_SNMP | 14 -NF9_V4_NEXT_HOP | 15 -NF9_SRC_AS | 16 -NF9_DST_AS | 17 -NF9_BGP_V4_NEXT_HOP | 18 -NF9_LAST_SWITCHED | 21 -NF9_FIRST_SWITCHED | 22 -NF9_OUT_BYTES | 23 -NF9_OUT_PKTS | 24 -NF9_IPV6_SRC_ADDR | 27 -NF9_IPV6_DST_ADDR | 28 -NF9_IPV6_SRC_MASK | 29 -NF9_IPV6_DST_MASK | 30 -NF9_IPV6_FLOW_LABEL | 31 -NF9_ICMP_TYPE | 32 -NF9_SAMPLING_INTERVAL | 34 -NF9_SAMPLING_ALGORITHM | 35 -NF9_ENGINE_TYPE | 38 -NF9_ENGINE_ID | 39 -NF9_FLOW_SAMPLER_ID | 48 -FLOW_SAMPLER_MODE | 49 -NF9_FLOW_SAMPLER_RANDOM_INTERVAL | 50 -NF9_MIN_TTL | 52 -NF9_MAX_TTL | 53 -NF9_IPV4_IDENT | 54 -NF9_DST_TOS | 55 -NF9_IN_SRC_MAC | 56 -NF9_OUT_DST_MAC | 57 -NF9_SRC_VLAN | 58 -NF9_DST_VLAN | 59 -NF9_DIRECTION | 61 -NF9_V6_NEXT_HOP | 62 -NF9_BPG_V6_NEXT_HOP | 63 -// NF9_V6_OPTION_HEADERS | 64 -NF9_MPLS_LABEL_1 | 70 -NF9_MPLS_LABEL_2 | 71 -NF9_MPLS_LABEL_3 | 72 -NF9_MPLS_LABEL_4 | 73 -NF9_MPLS_LABEL_5 | 74 -NF9_MPLS_LABEL_6 | 75 -NF9_MPLS_LABEL_7 | 76 -NF9_MPLS_LABEL_8 | 77 -NF9_MPLS_LABEL_9 | 78 -NF9_MPLS_LABEL_10 | 79 -NF9_IN_DST_MAC | 80 -NF9_OUT_SRC_MAC | 81 -NF9_FORWARDING_STATUS | 89 -NF9_BGP_ADJ_NEXT_AS | 128 -NF9_BGP_ADJ_PREV_AS | 129 - -### CISCO ASA NSEL extension - Network Security Event Logging__ -Tag | ID -----|--- -NF_F_FLOW_BYTES | 85 -NF_F_CONN_ID | 148 -NF_F_FLOW_CREATE_TIME_MSEC | 152 -NF_F_ICMP_TYPE | 176 -NF_F_ICMP_CODE | 177 -NF_F_ICMP_TYPE_IPV6 | 178 -NF_F_ICMP_CODE_IPV6 | 179 -NF_F_FWD_FLOW_DELTA_BYTES | 231 -NF_F_REV_FLOW_DELTA_BYTES | 232 -NF_F_FW_EVENT84 | 233 -NF_F_EVENT_TIME_MSEC | 323 -NF_F_INGRESS_ACL_ID | 33000 -NF_F_EGRESS_ACL_ID | 33001 -NF_F_FW_EXT_EVENT | 33002 -NF_F_USERNAME | 40000 -NF_F_XLATE_SRC_ADDR_IPV4 | 40001 -NF_F_XLATE_DST_ADDR_IPV4 | 40002 -NF_F_XLATE_SRC_PORT | 40003 -NF_F_XLATE_DST_PORT | 40004 -NF_F_FW_EVENT | 40005 - -### Cisco ASR 1000 series NEL extension - Nat Event Logging__ -Tag | ID -----|--- -NF_N_NAT_EVENT | 230 -NF_N_INGRESS_VRFID | 234 -NF_N_EGRESS_VRFID | 235 -NF_N_NAT_INSIDE_GLOBAL_IPV4 | 225 -NF_N_NAT_OUTSIDE_GLOBAL_IPV4 | 226 -NF_N_POST_NAPT_SRC_PORT | 227 -NF_N_POST_NAPT_DST_PORT | 228 - -### latency extensions for nfpcapd and nprobe__ -Tag | ID -----|--- -NF9_NPROBE_CLIENT_NW_DELAY_SEC | 57554 -NF9_NPROBE_CLIENT_NW_DELAY_USEC | 57555 -NF9_NPROBE_SERVER_NW_DELAY_SEC | 57556 -NF9_NPROBE_SERVER_NW_DELAY_USEC | 57557 -NF9_NPROBE_APPL_LATENCY_SEC | 57558 -NF9_NPROBE_APPL_LATENCY_USEC | 57559 - -32 and 64 bit counters are supported for any counters. However, internally -nfdump stores packets and bytes counters always as 64bit counters. -16 and 32 bit AS numbers are supported. - -Extensions: nfcapd supports a large number of v9 tags. In order to optimise -disk space and performance, v9 tags are grouped into a number of extensions -which may or may not be stored into the data file. Therefore the v9 templates configured on the exporter may be tuned with the collector. Only the tags common to both are stored into the data files. Extensions can be switch on/off by using the -T option. If you want to collect all data, use __-Tall__ +| Tag | ID | +| -------------------------------- | --- | +| NF9_IN_BYTES | 1 | +| IN_PACKETS | 2 | +| NF9_FLOWS_AGGR | 3 | +| NF9_IN_PROTOCOL | 4 | +| NF9_SRC_TOS | 5 | +| NF9_TCP_FLAGS | 6 | +| NF9_L4_SRC_PORT | 7 | +| NF9_IPV4_SRC_ADDR | 8 | +| NF9_SRC_MASK | 9 | +| NF9_INPUT_SNMP | 10 | +| NF9_L4_DST_PORT | 11 | +| NF9_IPV4_DST_ADDR | 12 | +| NF9_DST_MASK | 13 | +| NF9_OUTPUT_SNMP | 14 | +| NF9_V4_NEXT_HOP | 15 | +| NF9_SRC_AS | 16 | +| NF9_DST_AS | 17 | +| NF9_BGP_V4_NEXT_HOP | 18 | +| NF9_LAST_SWITCHED | 21 | +| NF9_FIRST_SWITCHED | 22 | +| NF9_OUT_BYTES | 23 | +| NF9_OUT_PKTS | 24 | +| NF9_IPV6_SRC_ADDR | 27 | +| NF9_IPV6_DST_ADDR | 28 | +| NF9_IPV6_SRC_MASK | 29 | +| NF9_IPV6_DST_MASK | 30 | +| NF9_IPV6_FLOW_LABEL | 31 | +| NF9_ICMP_TYPE | 32 | +| NF9_SAMPLING_INTERVAL | 34 | +| NF9_SAMPLING_ALGORITHM | 35 | +| NF9_ENGINE_TYPE | 38 | +| NF9_ENGINE_ID | 39 | +| NF9_FLOW_SAMPLER_ID | 48 | +| FLOW_SAMPLER_MODE | 49 | +| NF9_FLOW_SAMPLER_RANDOM_INTERVAL | 50 | +| NF9_MIN_TTL | 52 | +| NF9_MAX_TTL | 53 | +| NF9_IPV4_IDENT | 54 | +| NF9_DST_TOS | 55 | +| NF9_IN_SRC_MAC | 56 | +| NF9_OUT_DST_MAC | 57 | +| NF9_SRC_VLAN | 58 | +| NF9_DST_VLAN | 59 | +| NF9_DIRECTION | 61 | +| NF9_V6_NEXT_HOP | 62 | +| NF9_BPG_V6_NEXT_HOP | 63 | +| // NF9_V6_OPTION_HEADERS | 64 | +| NF9_MPLS_LABEL_1 | 70 | +| NF9_MPLS_LABEL_2 | 71 | +| NF9_MPLS_LABEL_3 | 72 | +| NF9_MPLS_LABEL_4 | 73 | +| NF9_MPLS_LABEL_5 | 74 | +| NF9_MPLS_LABEL_6 | 75 | +| NF9_MPLS_LABEL_7 | 76 | +| NF9_MPLS_LABEL_8 | 77 | +| NF9_MPLS_LABEL_9 | 78 | +| NF9_MPLS_LABEL_10 | 79 | +| NF9_IN_DST_MAC | 80 | +| NF9_OUT_SRC_MAC | 81 | +| NF9_FORWARDING_STATUS | 89 | +| NF9_BGP_ADJ_NEXT_AS | 128 | +| NF9_BGP_ADJ_PREV_AS | 129 | + +### CISCO ASA NSEL extension - Network Security Event Logging\_\_ + +| Tag | ID | +| -------------------------- | ----- | +| NF_F_FLOW_BYTES | 85 | +| NF_F_CONN_ID | 148 | +| NF_F_FLOW_CREATE_TIME_MSEC | 152 | +| NF_F_ICMP_TYPE | 176 | +| NF_F_ICMP_CODE | 177 | +| NF_F_ICMP_TYPE_IPV6 | 178 | +| NF_F_ICMP_CODE_IPV6 | 179 | +| NF_F_FWD_FLOW_DELTA_BYTES | 231 | +| NF_F_REV_FLOW_DELTA_BYTES | 232 | +| NF_F_FW_EVENT84 | 233 | +| NF_F_EVENT_TIME_MSEC | 323 | +| NF_F_INGRESS_ACL_ID | 33000 | +| NF_F_EGRESS_ACL_ID | 33001 | +| NF_F_FW_EXT_EVENT | 33002 | +| NF_F_USERNAME | 40000 | +| NF_F_XLATE_SRC_ADDR_IPV4 | 40001 | +| NF_F_XLATE_DST_ADDR_IPV4 | 40002 | +| NF_F_XLATE_SRC_PORT | 40003 | +| NF_F_XLATE_DST_PORT | 40004 | +| NF_F_FW_EVENT | 40005 | + +### Cisco ASR 1000 series NEL extension - Nat Event Logging\_\_ + +| Tag | ID | +| ---------------------------- | --- | +| NF_N_NAT_EVENT | 230 | +| NF_N_INGRESS_VRFID | 234 | +| NF_N_EGRESS_VRFID | 235 | +| NF_N_NAT_INSIDE_GLOBAL_IPV4 | 225 | +| NF_N_NAT_OUTSIDE_GLOBAL_IPV4 | 226 | +| NF_N_POST_NAPT_SRC_PORT | 227 | +| NF_N_POST_NAPT_DST_PORT | 228 | + +### latency extensions for nfpcapd and nprobe\_\_ + +| Tag | ID | +| ------------------------------- | ----- | +| NF9_NPROBE_CLIENT_NW_DELAY_SEC | 57554 | +| NF9_NPROBE_CLIENT_NW_DELAY_USEC | 57555 | +| NF9_NPROBE_SERVER_NW_DELAY_SEC | 57556 | +| NF9_NPROBE_SERVER_NW_DELAY_USEC | 57557 | +| NF9_NPROBE_APPL_LATENCY_SEC | 57558 | +| NF9_NPROBE_APPL_LATENCY_USEC | 57559 | + +32 and 64 bit counters are supported for any counters. However, internally nfdump stores packets and bytes counters always as 64bit counters. 16 and 32 bit AS numbers are supported. + +Extensions: nfcapd supports a large number of v9 tags. In order to optimise disk space and performance, v9 tags are grouped into a number of extensions which may or may not be stored into the data file. Therefore the v9 templates configured on the exporter may be tuned with the collector. Only the tags common to both are stored into the data files. Extensions can be switch on/off by using the -T option. If you want to collect all data, use **-Tall** ### Sampling -By default, the sampling rate is set to 1 (unsampled) or to -any given value specified by the -s cmd line option. If sampling information is found -in the netflow stream, it overwrites the default value. Sampling is automatically -recognised when announced in v9 option templates (tags #48, #49, #50 ), (tag #34, #35) -or in the unofficial v5 header hack. -Note: Not all platforms (or IOS versions) support exporting sampling information in -netflow data, even if sampling is configured. The number of bytes/packets in each -netflow record is automatically multiplied by the sampling rate. The total number of -flows is not changed as this is not accurate enough. (Small flows versus large flows) + +By default, the sampling rate is set to 1 (unsampled) or to any given value specified by the -s cmd line option. If sampling information is found in the netflow stream, it overwrites the default value. Sampling is automatically recognised when announced in v9 option templates (tags #48, #49, #50 ), (tag #34, #35) or in the unofficial v5 header hack. Note: Not all platforms (or IOS versions) support exporting sampling information in netflow data, even if sampling is configured. The number of bytes/packets in each netflow record is automatically multiplied by the sampling rate. The total number of flows is not changed as this is not accurate enough. (Small flows versus large flows) ### InfluxDB @@ -424,15 +347,9 @@ This is considered legacy and will be removed in future releases. Please conside Legacy: -You can send nfprofile stats data to an influxdb database. The data are the same of rrd files. -For enable this option you need libcurl dev package installed, use --enable-influxdb for configure the project and the nfprofile command should be invoked with option: -i . -Example: -i http://localhost:8086/write?db=mydb&u=user&p=pass -The parameters for auth (&u=user&p=pass) are optional. -Then you get the stats data on influxdb mydb in the measurement nfsen_stats. +You can send nfprofile stats data to an influxdb database. The data are the same of rrd files. For enable this option you need libcurl dev package installed, use --enable-influxdb for configure the project and the nfprofile command should be invoked with option: -i . Example: -i http://localhost:8086/write?db=mydb&u=user&p=pass The parameters for auth (&u=user&p=pass) are optional. Then you get the stats data on influxdb mydb in the measurement nfsen_stats. -For put the stats of live profile you need to apply a patch to nfsen (in extra/nfsen) and add in nfsen.conf the option: - $influxdb_url="http://mydbhost.local:8086/write?db=nfsen"; -as example I added a preconfigured grafana dashboard in extra/grafana/Nfsen_Stats.json . +For put the stats of live profile you need to apply a patch to nfsen (in extra/nfsen) and add in nfsen.conf the option: $influxdb_url="http://mydbhost.local:8086/write?db=nfsen"; as example I added a preconfigured grafana dashboard in extra/grafana/Nfsen_Stats.json . --- diff --git a/bin/Makefile.am b/bin/Makefile.am index 57d36b2c..534b589b 100755 --- a/bin/Makefile.am +++ b/bin/Makefile.am @@ -62,7 +62,7 @@ launch = launch.c launch.h lib_LTLIBRARIES = libnfdump.la libnfdump_la_SOURCES = $(output) $(util) $(filelzo) $(nffile) $(nflist) $(filter) $(exporter) -libnfdump_la_LDFLAGS = -release 1.6.23 +libnfdump_la_LDFLAGS = -release @VERSION@ nfdump_SOURCES = nfdump.c nfdump.h nfstat.c nfstat.h nfexport.c nfexport.h \ diff --git a/configure.ac b/configure.ac index 687fc6b4..60942c2e 100644 --- a/configure.ac +++ b/configure.ac @@ -3,7 +3,7 @@ AC_PREREQ([2.69]) AC_REVISION($Revision: 244 $)dnl -AC_INIT([nfdump],[1.6.23],[peter@people.ops-trust.net]) +AC_INIT([nfdump],[1.6.24],[peter@people.ops-trust.net]) AC_CONFIG_HEADERS([config.h]) AM_INIT_AUTOMAKE([subdir-objects])