forked from sandflysecurity/sandfly-entropyscan
-
Notifications
You must be signed in to change notification settings - Fork 1
/
fileutils.go
217 lines (174 loc) · 5.76 KB
/
fileutils.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
// Sandfly entropyscan file utilities to calculate entropy, crypto hashes, etc
package main
/*
This utility will help find packed or encrypted files or processes on a Linux system by calculating the entropy
to see how random they are. Packed or encrypted malware often appears to be a very random executable file and this
utility can help identify potential intrusions.
You can calculate entropy on all files, or limit the search just to Linux ELF executables that have an entropy of
your threshold. Linux processes can be scanned as well automatically.
Sandfly Security produces an agentless endpoint detection and incident response platform (EDR) for Linux. You can
find out more about how it works at: https://www.sandflysecurity.com
MIT License
Copyright (c) 2019-2022 Sandfly Security Ltd.
https://www.sandflysecurity.com
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of
the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Version: 1.1.1
Author: @SandflySecurity
*/
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"io"
"math"
"os"
)
const (
// Max file size for entropy, etc. is 2GB
constMaxFileSize = 2147483648
// Chunk of data size to read in for entropy calc
constMaxEntropyChunk = 256000
// Need 4 bytes to determine basic ELF type
constMagicNumRead = 4
// Magic number for basic ELF type
constMagicNumElf = "7f454c46"
)
type ErrNotRegularFile struct {
Path string
}
func (e *ErrNotRegularFile) Error() string {
return fmt.Sprintf("file '%s' is not a regular file", e.Path)
}
func NewErrNotRegularFile(path string) *ErrNotRegularFile {
return &ErrNotRegularFile{Path: path}
}
type ErrFileTooLarge struct {
Path string
Size int64
Max int64
}
func (e *ErrFileTooLarge) Error() string {
return fmt.Sprintf("file size of '%s' is too large (%d bytes) to calculate entropy (max allowed: %d bytes)",
e.Path, e.Size, e.Max)
}
func NewErrFileTooLarge(path string, size int64) *ErrFileTooLarge {
return &ErrFileTooLarge{Path: path, Size: size, Max: constMaxFileSize}
}
var ErrNoPath = fmt.Errorf("no path provided")
var elfType []byte
func init() {
var err error
if elfType, err = hex.DecodeString(constMagicNumElf); err != nil {
// this should never happen
panic(fmt.Errorf("couldn't decode ELF magic number constant: %w", err))
}
if len(elfType) > constMagicNumRead {
// this should never happen
panic(fmt.Errorf("elf magic number string is longer than magic number read bytes"))
}
}
func preCheckFilepath(path string) (*os.File, int64, error) {
if path == "" {
return nil, 0, ErrNoPath
}
f, err := os.Open(path)
if err != nil {
return nil, 0, fmt.Errorf("couldn't open '%s': %w", path, err)
}
fStat, err := f.Stat()
if err != nil {
if f != nil {
_ = f.Close()
}
return f, 0, err
}
if !fStat.Mode().IsRegular() {
_ = f.Close()
return f, 0, NewErrNotRegularFile(path)
}
if fStat.Size() == 0 {
_ = f.Close()
return f, fStat.Size(), fmt.Errorf("file '%s' is zero size", path)
}
return f, fStat.Size(), nil
}
// IsFileElf will reead the magic bytes from the passed file and determine if it is an ELF file.
func IsFileElf(path string) (isElf bool, err error) {
var fSize int64
var f io.ReadCloser
if f, fSize, err = preCheckFilepath(path); err != nil {
return false, err
}
defer func() {
_ = f.Close()
}()
// Too small to be ELF
if fSize < constMagicNumRead {
return false, fmt.Errorf("file '%s' is too small to be an ELF file", path)
}
return IsElf(f)
}
func IsElf(f io.Reader) (isElf bool, err error) {
var hexData [constMagicNumRead]byte
var n int
if n, err = f.Read(hexData[:]); err != nil {
return false, fmt.Errorf("read failure during ELF check: %w", err)
}
if n != constMagicNumRead {
return false, fmt.Errorf("%w: undersized read during ELF check", io.ErrUnexpectedEOF)
}
return bytes.Equal(hexData[:len(elfType)], elfType), nil
}
func Entropy(f io.Reader, size int64) (entropy float64, err error) {
dataBytes := make([]byte, constMaxEntropyChunk)
byteCounts := make([]int, 256)
for {
numBytesRead, readErr := f.Read(dataBytes)
if errors.Is(readErr, io.EOF) {
break
}
if readErr != nil {
return 0, readErr
}
// For each byte of the data that was read, increment the count
// of that number of bytes seen in the file in our byteCounts
// array
for i := 0; i < numBytesRead; i++ {
byteCounts[int(dataBytes[i])]++
}
}
for i := 0; i < 256; i++ {
px := float64(byteCounts[i]) / float64(size)
if px > 0 {
entropy += -px * math.Log2(px)
}
}
// Returns rounded to nearest two decimals.
return math.Round(entropy*100) / 100, nil
}
// FileEntropy calculates entropy of a file.
func FileEntropy(path string) (entropy float64, err error) {
var size int64
var f io.ReadCloser
if f, size, err = preCheckFilepath(path); err != nil {
return 0, err
}
defer func() {
_ = f.Close()
}()
if size > int64(constMaxFileSize) {
return 0, NewErrFileTooLarge(path, size)
}
return Entropy(f, size)
}