From 6334c913fdf64a71e7e9a77ce67733b754a877f4 Mon Sep 17 00:00:00 2001 From: Sudhi Herle Date: Wed, 29 Apr 2020 17:51:50 -0700 Subject: [PATCH] Python3 changes; updates to work with newer kernels and OSes. --- abspath | 2 +- aes-test.sh | 95 ++++--- aes.py | 465 +++++++++++++++++++---------------- comic2pdf | 6 +- dos2unix.py | 4 +- find-usb-disk | 181 ++++++++++++-- finddup | 46 ++-- hexlify | 149 +++++++---- kern-build | 47 ++-- kill.py | 308 +++++++++++------------ linktest.py | 22 +- mk-raw-vmware.py | 6 +- openbsd-randmac.sh | 2 +- osx-randmac.sh | 11 +- pingsubnet.py | 599 ++++++++++++++++++++++++--------------------- randmac | 4 +- randpass | 4 +- realpath | 2 +- rm.py | 10 +- tolower | 14 +- 20 files changed, 1137 insertions(+), 840 deletions(-) diff --git a/abspath b/abspath index e6c6c87..525fee4 100755 --- a/abspath +++ b/abspath @@ -3,4 +3,4 @@ import os.path, sys for a in sys.argv[1:]: - print os.path.abspath(a) + print(os.path.abspath(a)) diff --git a/aes-test.sh b/aes-test.sh index 2c57a53..0c8576d 100755 --- a/aes-test.sh +++ b/aes-test.sh @@ -4,10 +4,13 @@ # Author: Sudhi Herle # License: Public Domain +TMP=$HOME/tmp/aes AES=./aes.py [ -n "$PYTHON" ] && AES="$PYTHON $AES" #AES='python3.4 ./aes.py' +mkdir -p $TMP || exit 1 + uname=`uname` case $uname in Linux) MD5=md5sum ;; @@ -43,10 +46,13 @@ xverify() { testsz() { #set -x local sz=$1; shift + local bsiz=$1; shift + + [ -n "$bsiz" ] && bsiz="-B $bsiz" - local inf=/tmp/in$sz - local enc=/tmp/enc$sz - local dec=/tmp/dec$sz + local inf=${TMP}/in$sz + local enc=${TMP}/enc$sz + local dec=${TMP}/dec$sz if [ $sz -gt 0 ]; then dd if=/dev/urandom of=$inf bs=$sz count=1 2>/dev/null @@ -54,23 +60,28 @@ testsz() { touch $inf fi - begin "Testing file $sz .." - FX=abcdef $AES -k FX encrypt $inf -o $enc || exit 1 - FX=abcdef $AES -k FX test $enc || exit 1 - FX=abcdef $AES -k FX decrypt $enc -o $dec || exit 1 + echo "File size $sz:" + + begin " regular file I/O ..." + FX=abcdef $AES -k FX $bsiz encrypt $inf -o $enc || exit 1 + FX=abcdef $AES -k FX $bsiz test $enc || exit 1 + FX=abcdef $AES -k FX $bsiz decrypt $enc -o $dec || exit 1 xverify $inf $dec || exit 1 - begin "Testing inplace $sz .." - cp $inf $enc - FX=abcdef $AES -k FX encrypt $enc -o $enc || exit 1 + begin " streaming I/O ..." + cat $inf | FX=abcdef $AES -k FX $bsiz encrypt | cat > $enc + [ $? -ne 0 ] && end Fail && exit 1 - cp $enc $dec - FX=abcdef $AES -k FX decrypt $dec -o $dec || exit 1 + cat $enc | FX=abcdef $AES -k FX $bsiz decrypt | cat > $dec + [ $? -ne 0 ] && end Fail && exit 1 xverify $inf $dec || exit 1 - begin "Testing stdio $sz .." - FX=abcdef $AES -k FX encrypt < $inf > $enc || exit 1 - FX=abcdef $AES -k FX decrypt < $enc > $dec || exit 1 + begin " inplace ..." + cp $inf $enc + FX=abcdef $AES -k FX $bsiz encrypt $enc -o $enc || exit 1 + + cp $enc $dec + FX=abcdef $AES -k FX $bsiz decrypt $dec -o $dec || exit 1 xverify $inf $dec || exit 1 @@ -79,32 +90,41 @@ testsz() { # Full suite of functional tests basic() { - local sz=3791 - local inf=/tmp/in$sz - local enc=/tmp/enc$sz - local dec=/tmp/dec$sz + #set -x + local sz=$(randsz) + local inf=${TMP}/in$sz + local enc=${TMP}/enc$sz + local dec=${TMP}/dec$sz local enc2=${enc}.2 - local szskip=$(( sz / 2 )) - local badcount=$(( sz / 4 )) + local szskip=$(( $sz / 2 )) + local badcount=$(( $sz / 4 )) + local bsize=$(( $sz / 2 )) dd if=/dev/urandom of=$inf bs=$sz count=1 2>/dev/null || exit 1 - begin "Testing basic functions with size $sz .." + echo "Basic tests with size $sz .." + begin " basic functions ..." FX=abcdef $AES -k FX encrypt $inf -o $enc || exit 1 FX=abcdef $AES -k FX test $enc || exit 1 FX=abcdef $AES -k FX decrypt $enc -o $dec || exit 1 xverify $inf $dec || exit 1 + begin " basic functions (bufsize $bsize) ..." + FX=abcdef $AES -k FX -B $bsize encrypt $inf -o $enc || exit 1 + FX=abcdef $AES -k FX -B $bsize test $enc || exit 1 + FX=abcdef $AES -k FX -B $bsize decrypt $enc -o $dec || exit 1 + xverify $inf $dec || exit 1 + # Bad password should fail - FX=abcxyz $AES -k FX test $enc 2>/dev/null && exit 1 + FX=abcxyz $AES -k FX test $enc 2>/dev/null && end Fail && exit 1 # Corrupted enc file should fail cp $enc $enc2 || exit 1 - dd if=/dev/urandom of=$enc2 count=1 bs=$(($szskip / 3)) \ + dd if=/dev/urandom of=$enc2 count=1 bs=$(($szskip * 2)) \ seek=$szskip conv=notrunc 2>/dev/null || exit 1 - FX=abcdef $AES -k FX test $enc2 2>/dev/null && exit 1 + FX=abcdef $AES -k FX test $enc2 2>/dev/null && end Fail && exit 1 - begin "Testing basic inplace with size $sz .." + begin " inplace ..." cp $inf $enc FX=abcdef $AES -k FX encrypt $enc -o $enc || exit 1 @@ -112,22 +132,22 @@ basic() { FX=abcdef $AES -k FX decrypt $dec -o $dec || exit 1 xverify $inf $dec || exit 1 - begin "Testing basic stdio with size $sz .." - FX=abcdef $AES -k FX encrypt < $inf > $enc || exit 1 - FX=abcdef $AES -k FX decrypt < $enc > $dec || exit 1 - xverify $inf $dec || exit 1 - + begin " stream input and output ..." + cat $inf | FX=abcdef $AES -k FX encrypt | cat > $enc + [ $? -ne 0 ] && end Fail && exit 1 - # Now test corrupted files and bad mac + cat $enc | FX=abcdef $AES -k FX decrypt | cat > $dec + [ $? -ne 0 ] && end Fail && exit 1 + xverify $inf $dec || exit 1 rm -f $inf $enc $dec $enc2 } -# Generate random ints between [100, 100000) +# Generate random ints between x and y randsz() { - local x=101 - local y=100000 + local x=78329 + local y=900000 local r=$(( $y - $x + 1)) local n=0 @@ -143,7 +163,7 @@ randsz() { } -trap 'exit 0' INT TERM QUIT +trap "rm -rf $TMP; exit 0" INT TERM QUIT echo "Testing $AES .." basic @@ -158,6 +178,7 @@ done n=8 while [ $n -gt 0 ]; do n=$(( $n - 1 )) - testsz $(randsz) + sz=$(randsz) + testsz $sz $(( $sz / 8 )) done diff --git a/aes.py b/aes.py index bb25b30..2b4fed5 100755 --- a/aes.py +++ b/aes.py @@ -16,13 +16,16 @@ # - Handles in-place encryption/decryption - i.e., if same file name is used as # the output file, the program does the "right thing" # - AES in CTR mode; HMAC-SHA256 for integrity protection +# - Input is chunked into BUF_SIZE blocks and each chunk is individually HMAC'd +# - chunk-size & HMAC written as header of ciphertext chunk # - Encrypt-then-hash # - Uses PBKDF2 to derive two keys (one for hmac and one for cipher) # - Single random salt used for KDF +# - IV for AES-CTR is derived from salt (first 16 bytes of SHA512 of salt) +# - User supplied passphrase is hashed with SHA512 before sending to KDF. # - A password verifier calculated as SHA256 of KDF output; this is used to # verify correct password at decryption time. # - Verifier, salts, algo choice, KDF params are written as header (struct.pack) -# - HMAC written as trailer of encrypted stream # - Important information is captured in the header of the encrypted stream. # Thus, as long as header format stays same, future versions of the program # can still decrypt files encrypted with older version of script. @@ -84,130 +87,94 @@ F_HDRSZ = struct.calcsize(F_HDRFMT) # --- --- --- +def main(): + global VERSION, BUF_SIZE -def warn(fmt, *args): - sf = fmt % args if args else fmt - if not sf.endswith('\n'): - sf += '\n' - - sys.stderr.write(sf) - sys.stderr.flush() + ap = argparse.ArgumentParser(description="""Portable encryption/decryption of file. + Encryption uses AES-256 in CTR mode. Encrypted data is authenticated with + HMAC-SHA-256 (Encrypt-then-MAC). The user supplied passphrase is used to + derive encryption and HMAC keys using the PBKDF2 function. -def die(fmt, *args): - warn(fmt, *args) - sys.exit(1) + If both input and output file names are identical, then the script + assumes in-place transformation and uses temporary files for the + operation. + """) + ap.add_argument("-o", "--output", type=str, default=None, metavar='F', + help="write output to file 'F' [STDOUT]") + ap.add_argument("-k", "--env-pass", type=str, default=None, metavar='E', + help="read password from environment variable 'E' []") + ap.add_argument("-B", "--io-buf-size", type=int, default=BUF_SIZE, metavar='N', + dest="bufsize", + help="Do I/O in chunks of 'N' bytes [%(default)d]") + ap.add_argument('-V', "--version", action='version', version=VERSION) -def equal(a,b): - """Constant time equals for iterables - For PY3K, bytes objects are already integers. So, no need for ord() - """ + ap.add_argument("op", choices=['encrypt', 'decrypt', 'test'], + help="operation to perform") + ap.add_argument("infile", nargs='?', help="input file to encrypt|decrypt|test [STDIN]") - if len(a) != len(b): return False + args = ap.parse_args() - am = a if PY3K else map(ord, a) - bm = b if PY3K else map(ord, b) + if args.bufsize <= 0: + die("Bufsize %d must be a positive number", args.bufsize) - v = 0 - for x, y in zip(am, bm): - v |= x ^ y - return True if v == 0 else False + BUF_SIZE = args.bufsize + # Don't ask for password twice if we are decrypting. + pwd = get_pass(args.env_pass, tries=2 if args.op == "encrypt" else 1) -def randbytes(n): - """Get a random string of n bytes length""" - return os.urandom(n) + if args.op == "test": + infd = openif(args.infile, sys.stdin, 'rb') + outfd = nullwriter() + cf = cipher.decbegin(infd, pwd) + cf.dec(outfd) + infd.close() + return -def randchars(n): - """Get a random string of n printable chars""" - global CHARSET + #print "PASS=%s ENV=%s" % (pwd, args.env_pass) - random.shuffle(CHARSET) - x = random.sample(CHARSET, n) + # Verify for in place operation + inplace = False + if samefile(args.infile, args.output): + inplace = True + args.infile = abspath(normpath(args.infile)) + args.output = args.infile + randchars(8) - return ''.join(x) + infd = openif(args.infile, sys.stdin, 'rb') + outfd = openif(args.output, sys.stdout, 'wb') -def dump(fmt, *args): - """Dump one or more variables containing binary data""" - if args: - v = ( hexlify(a) for a in args ) - x = fmt % tuple(v) + if args.op == "encrypt": + cf = cipher('AES', 'SHA256', 'PBKDF2', KEYLEN, SALTLEN, KEY_ROUNDS) + cf.encbegin(pwd, outfd) + cf.enc(infd) else: - x = fmt - - if not x.endswith('\n'): - x += '\n' - - sys.stderr.write(x) - sys.stderr.flush() - -def hashbytes(*a): - """Hash a and return SHA256""" - h = HASH.new() - for x in a: - lb = pack("> I", len(x)) - h.update(lb) - h.update(x) + cf = cipher.decbegin(infd, pwd) + cf.dec(outfd) - return h.digest() + outfd.close() + infd.close() -def sha512(pw): - """hash the password once; expands shorter passwords""" - h = SHA512.new() - lb = pack("> I", len(pw)) - h.update(lb) - h.update(pw) - return h.digest() + if inplace: + tmp = args.infile + randchars(8) -def get_pass(env=None, tries=1): - """Read password from console or environment var""" - global PY3K + # Save the orig file temporarily + try: + os.rename(args.infile, tmp) + except Exception as ex: + os.unlink(args.output) + die("Operation failed; Unable to create temporary restore point\n\t%s", str(ex)) - if PY3K: - pwenc = lambda x: x.encode('utf-8') + try: + os.rename(args.output, args.infile) + except Exception as ex: + os.rename(tmp, args.infile) + os.unlink(args.output) + die("Operation failed; unable to rename transformed file back to original\n\t%s", str(ex)) else: - pwenc = lambda x: x - - if env: - pw = os.environ.get(env, None) - if not pw: - die("Environment var %s is empty?", pw) - - return sha512(pwenc(pw)) - - if tries == 1: - pw = getpass.getpass("Enter password: ", sys.stderr) - return sha512(pwenc(pw)) - - while tries > 0: - tries -= 1 - pw1 = getpass.getpass("Enter password: ", sys.stderr) - pw2 = getpass.getpass("Re-enter password: ", sys.stderr) - if pw1 == pw2: - return sha512(pwenc(pw1)) - - warn("Passwords don't match. Try again..") - - die("Too many mistakes. Bye!") - - -def kdfprf(p, s): - """Pseudo-random-function used by PBKDF2. We use HMAC-SHA512.""" - - h0 = HMAC.new(p, digestmod=SHA512) - h0.update(s) - return h0.digest() - - -def pwverifier(k1, k2): - """Generate password verifier from two KDF derived keys""" + os.unlink(tmp) - s = HASH.new() - s.update(k1) - s.update(k2) - return s.digest() class cipher: @@ -235,27 +202,35 @@ def __init__(self, encalgo, hashalgo, kdfalgo, keylen, saltlen, rounds, bufsize= self.outfd = None - def derivekeys(self, pw, s1): + def derivekeys(self, pw, salt): """Derive the keys needed and return a verifier""" blksize = self.cipher.block_size # Derive two keys worth of key-material: one for Enc and one for HMAC - kx = self.kdf(pw, s1, self.keylen * 2, self.rounds, kdfprf) + kx = self.kdf(pw, salt, self.keylen * 2, self.rounds, kdfprf) k1 = kx[:self.keylen] k2 = kx[self.keylen:] # initial counter is derived from the salt - iv0 = sha512(s1)[:16] + iv0 = sha512(salt)[:16] iv = int(hexlify(iv0), 16) ctr = Counter.new(8 * blksize, initial_value=iv) - self.H = HMAC.new(k1, digestmod=self.shash) + self.k1 = k1 self.C = self.cipher.new(k2, self.cipher.MODE_CTR, counter=ctr) return pwverifier(k1, k2) + def hmac(self, chunkoff): + """Return new HMAC instance""" + + z = pack(">Q", chunkoff) + k = sha256(self.k1 + z) + + return HMAC.new(k, digestmod=self.shash) + @classmethod def decbegin(klass, infd, pw): """Initialize decryption by reading from 'infd'. @@ -285,25 +260,22 @@ def decbegin(klass, infd, pw): if bs == 0 or bs > (1024 * 1048576): die("Invalid buffer size in header") + # XXX If we change algorithms -- replace HASHSIZE with correct size + # obtained from header algorithm + vf = "> %ds %ds" % (sl, HASHSIZE) + sln = sl + HASHSIZE vhdr = infd.read(sln) if len(vhdr) < sln: die("Header incomplete, expected %d bytes, saw %d bytes" % (sln, len(vhdr))) - # XXX If we change algorithms -- replace HASHSIZE with correct size - # obtained from header algorithm - vf = "> %ds %ds" % (sl, HASHSIZE) - vv = unpack(vf, vhdr) - cf = cipher('AES', 'SHA256', 'PBKDF2', kl, sl, rr, bufsize=bs) + + vv = unpack(vf, vhdr) vx = cf.derivekeys(pw, vv[0]) if not equal(vx, vv[1]): die("Password mismatch. Aborting!") - hdr = fhdr + vhdr - lb = pack(">I", len(hdr)) - cf.H.update(lb) - cf.H.update(hdr) cf.infd = infd return cf @@ -320,10 +292,6 @@ def encbegin(self, pw, outfd): h2 = pack(vf, s1, v) hdr = h1 + h2 - - lb = pack(">I", len(hdr)) - self.H.update(lb) - self.H.update(hdr) self.outfd.write(hdr) @@ -337,54 +305,61 @@ def enc(self, infd): if not buf: break - eb = self.C.encrypt(buf) - self.H.update(eb) - outfd.write(eb) - n += len(eb) - - # Finally the total length of the data we've read so far. - z = pack(">Q", n) - self.H.update(z) - outfd.write(self.H.digest()) + r = len(buf) + z = pack(">I", r) + h = self.hmac(n) + e = self.C.encrypt(buf) + h.update(z) + h.update(e) + hm = z + h.digest() + #print "off=%d; sz=%d hash=%s" % (n, r, hexlify(hm[4:])) + outfd.write(hm) + outfd.write(e) + n += len(e) + n += len(hm) def dec(self, outfd): """Decrypt by reading the input and writing to outfd""" - n = 0 + # Each chunk we decrypt must have 4 byte plaintext len plus HMAC of the + # chunk. + hdrsz = 4 + HASHSIZE infd = self.infd - prev = infd.read(self.bufsize) - while len(prev) == self.bufsize: - buf = infd.read(self.bufsize) - if not buf: + off = 0 + + while True: + hdr = infd.read(hdrsz) + if not hdr: break - n += len(prev) - self.H.update(prev) - dbuf = self.C.decrypt(prev) - outfd.write(dbuf) + if len(hdr) != hdrsz: + die("Incomplete chunk-header near offset %d; exp %d, saw %d bytes", off, hdrsz, len(hdr)) - prev = buf + h = self.hmac(off) + sz = unpack(">I", hdr[:4])[0] + hm = hdr[4:] - # Last block has the mac. Remove it before decrypting the rest of the content - if len(prev) < HASHSIZE: - die("Corrupt file? Last block too small (%d bytes)" % len(prev)) + if sz == 0 or sz > self.bufsize: + die("malformed chunk-header near offset %d; expected size %d, saw %d", off, sz, self.bufsize) - file_mac = prev[-HASHSIZE:] - ebuf = prev[:-HASHSIZE] + #print "off=%d; sz=%d hash=%s" % (off, sz, hexlify(hm)) - n += len(ebuf) - self.H.update(ebuf) + off += hdrsz + ebuf = infd.read(sz) + if len(ebuf) != sz: + die("Incomplete chunk-body near offset %d; exp %d, saw %d bytes", off, sz, len(ebuf)) - z = pack(">Q", n) - self.H.update(z) - mac = self.H.digest() + h.update(hdr[:4]) + h.update(ebuf) + d = h.digest() + if not equal(d, hm): + die("Bad chunk near offset %d: MAC mismatch", off) - if not equal(file_mac, mac): - die("Corrupt file? MAC mismatch") + buf = self.C.decrypt(ebuf) + outfd.write(buf) + off += sz - dbuf = self.C.decrypt(ebuf) - outfd.write(dbuf) class nullwriter: @@ -456,87 +431,151 @@ def openif(nm, fd, mod): return open(nm, mod, BUF_SIZE) if nm else os.fdopen(fd.fileno(), mod) -def main(): - global VERSION +def seekable(fd): + """Return true if fd is a seekable file object""" - ap = argparse.ArgumentParser(description="""Portable encryption/decryption of file. - Encryption uses AES-256 in CTR mode. Encrypted data is authenticated with - HMAC-SHA-256 (Encrypt-then-MAC). The user supplied passphrase is used to - derive encryption and HMAC keys using the PBKDF2 function. + try: + # try to go to end of file + z = fd.seek(0, 2) - If both input and output file names are identical, then the script - assumes in-place transformation and uses temporary files for the - operation. - """) + except: + return False + finally: + fd.seek(0) - ap.add_argument("-o", "--output", type=str, default=None, metavar='F', - help="write output to file 'F' [STDOUT]") - ap.add_argument("-k", "--env-pass", type=str, default=None, metavar='E', - help="read password from environment variable 'E' []") - ap.add_argument('-V', "--version", action='version', version=VERSION) + return True +def warn(fmt, *args): + sf = fmt % args if args else fmt + if not sf.endswith('\n'): + sf += '\n' - ap.add_argument("op", choices=['encrypt', 'decrypt', 'test'], - help="operation to perform") - ap.add_argument("infile", nargs='?', help="input file to encrypt|decrypt|test [STDIN]") + sys.stderr.write(sf) + sys.stderr.flush() - args = ap.parse_args() +def die(fmt, *args): + warn(fmt, *args) + sys.exit(1) - # Don't ask for password twice if we are decrypting. - pwd = get_pass(args.env_pass, tries=2 if args.op == "encrypt" else 1) +def equal(a,b): + """Constant time equals for iterables - if args.op == "test": - infd = openif(args.infile, sys.stdin, 'rb') - outfd = nullwriter() - cf = cipher.decbegin(infd, pwd) - cf.dec(outfd) - infd.close() - return + For PY3K, bytes objects are already integers. So, no need for ord() + """ - #print "PASS=%s ENV=%s" % (pwd, args.env_pass) + if len(a) != len(b): return False - # Verify for in place operation - inplace = False - if samefile(args.infile, args.output): - inplace = True - args.infile = abspath(normpath(args.infile)) - args.output = args.infile + randchars(8) + am = a if PY3K else map(ord, a) + bm = b if PY3K else map(ord, b) - infd = openif(args.infile, sys.stdin, 'rb') - outfd = openif(args.output, sys.stdout, 'wb') + v = 0 + for x, y in zip(am, bm): + v |= x ^ y - if args.op == "encrypt": - cf = cipher('AES', 'SHA256', 'PBKDF2', KEYLEN, SALTLEN, KEY_ROUNDS) - cf.encbegin(pwd, outfd) - cf.enc(infd) + return True if v == 0 else False + + +def randbytes(n): + """Get a random string of n bytes length""" + return os.urandom(n) + +def randchars(n): + """Get a random string of n printable chars""" + global CHARSET + + random.shuffle(CHARSET) + x = random.sample(CHARSET, n) + + return ''.join(x) + +def dump(fmt, *args): + """Dump one or more variables containing binary data""" + if args: + v = ( hexlify(a) for a in args ) + x = fmt % tuple(v) else: - cf = cipher.decbegin(infd, pwd) - cf.dec(outfd) + x = fmt - outfd.close() - infd.close() + if not x.endswith('\n'): + x += '\n' - if inplace: - tmp = args.infile + randchars(8) + sys.stderr.write(x) + sys.stderr.flush() - # Save the orig file temporarily - try: - os.rename(args.infile, tmp) - except Exception as ex: - os.unlink(args.output) - die("Operation failed; Unable to create temporary restore point\n\t%s", str(ex)) +def hashbytes(*a): + """Hash a and return SHA256""" + h = HASH.new() + for x in a: + lb = pack("> I", len(x)) + h.update(lb) + h.update(x) - try: - os.rename(args.output, args.infile) - except Exception as ex: - os.rename(tmp, args.infile) - os.unlink(args.output) - die("Operation failed; unable to rename transformed file back to original\n\t%s", str(ex)) + return h.digest() + +def sha512(pw): + """hash the password once; expands shorter passwords""" + h = SHA512.new() + lb = pack("> I", len(pw)) + h.update(lb) + h.update(pw) + return h.digest() + +def sha256(s): + h = SHA256.new() + lb = pack("> I", len(s)) + h.update(lb) + h.update(s) + return h.digest() + +def get_pass(env=None, tries=1): + """Read password from console or environment var""" + global PY3K + + if PY3K: + pwenc = lambda x: x.encode('utf-8') else: - os.unlink(tmp) + pwenc = lambda x: x + + if env: + pw = os.environ.get(env, None) + if not pw: + die("Environment var %s is empty?", pw) + + return sha512(pwenc(pw)) + + if tries == 1: + pw = getpass.getpass("Enter password: ", sys.stderr) + return sha512(pwenc(pw)) + + while tries > 0: + tries -= 1 + pw1 = getpass.getpass("Enter password: ", sys.stderr) + pw2 = getpass.getpass("Re-enter password: ", sys.stderr) + if pw1 == pw2: + return sha512(pwenc(pw1)) + + warn("Passwords don't match. Try again..") + + die("Too many mistakes. Bye!") + + +def kdfprf(p, s): + """Pseudo-random-function used by PBKDF2. We use HMAC-SHA512.""" + + h0 = HMAC.new(p, digestmod=SHA512) + h0.update(s) + return h0.digest() +def pwverifier(k1, k2): + """Generate password verifier from two KDF derived keys""" + + s = HASH.new() + s.update(k1) + s.update(k2) + return s.digest() + main() diff --git a/comic2pdf b/comic2pdf index 15998f1..cee7420 100755 --- a/comic2pdf +++ b/comic2pdf @@ -130,7 +130,7 @@ def img2pdf(pdf, imgs): c.save() - except Exception, ex: + except Exception as ex: warn("Exception while converting %s: %s", fn, ex) def process(fn): @@ -149,7 +149,7 @@ def process(fn): bn, ex = splitext(fn) pdf = bn + '.pdf' - print "%s --> %s" % (basename(fn), basename(pdf)) + print("%s --> %s" % (basename(fn), basename(pdf))) img2pdf(pdf, imgs) rmtree(dn) @@ -170,7 +170,7 @@ def main(): dn = basename(fn) pdf = dn + '.pdf' - print "%s --> %s" % (dn, basename(pdf)) + print("%s --> %s" % (dn, basename(pdf))) img2pdf(pdf, imgs) diff --git a/dos2unix.py b/dos2unix.py index 4641c16..14406df 100755 --- a/dos2unix.py +++ b/dos2unix.py @@ -40,7 +40,7 @@ def transcode(fn, verbose=False): if mod: if verbose: - print "%s" % fn + print("%s" % fn) bak = fn + '.orig' os.rename(fn, bak) @@ -77,4 +77,4 @@ def fixup(dn, verbose=False): elif os.path.isdir(f): fixup(f, True) else: - print "Hmm. %s is not a file or dir" % f + print("Hmm. %s is not a file or dir" % f) diff --git a/find-usb-disk b/find-usb-disk index b343378..9962f6d 100755 --- a/find-usb-disk +++ b/find-usb-disk @@ -1,34 +1,177 @@ #! /bin/bash +# Find and print info about USB devices on linux. +# (c) 2005-2018 Sudhi Herle +# License: GPLv2 -#set -x -find_usb_disk() { - disk= - entries=`ls /sys/block | sort` - for e in $entries; do - d=/sys/block/$e - n=`readlink $d | grep usb | wc -l` - if [ $n -eq 0 ]; then +Z=`basename $0` +e=echo +Verbose=0 +Partition=0 + + +die() { + echo "$Z: $@" 1>&2 + exit 0 +} + +warn() { + echo "$Z: $@" 1>&2 +} + +Uname=$(uname) +case $Uname in + Linux*) + ;; + + *) die "This only works on Linux!" + ;; +esac + +Fdisk=$(type -p fdisk) +[ -z $Fdisk ] && warn "Can't find fdisk; No partition info .." + + +main() { + local ac_prev= + local args= + local ac_option= + local ac_optarg= + + for ac_option + do + shift + + if [ -n "$ac_prev" ]; then + eval "$ac_prev=\$ac_option" + ac_prev= continue fi - i=`basename $d` - dev=/dev/$i - sys=/sys/block/$i + case "$ac_option" in + -*=*) ac_optarg=`echo "$ac_option" | sed 's/[-_a-zA-Z0-9]*=//'` ;; + *) ac_optarg= ;; + esac + + + case "$ac_option" in + --help|-h|--hel|--he|--h) + usage; + ;; + + -v|--verbose) + Verbose=1 + ;; + + -p|--show-partitions) + Partition=1 + ;; + + --debug|-x) + set -x + ;; + + *) # first non option terminates option processing. + # we gather all remaining args and bundle them up. + args="$args $ac_option" + for xx + do + args="$args $xx" + done + break + ;; + esac + done + + find_usb_disk +} + +find_usb_disk() { + local root=/dev/disk/by-path + for e in $(ls $root | grep usb); do + local fp=$root/$e + local nm=$(readlink $fp) + + nm=$(basename $nm) + local disk=${nm%%[1-9]*} + [ $nm == $disk ] && print_disk "/dev/$disk" /sys/block/$disk + done + return 0 +} + + +# Print details of a disk +print_disk() { + local dev=$1 + local sysfs=$2 + local sz=$(cat $sysfs/size) - rm=`cat $sys/removable` - if [ $rm -eq 0 ]; then - continue # ignore fixed disks + # XXX Implcitly convert to GB + # size is in sectors (512 bytes). + sz=$(humanize $sz) + + if [ $Verbose -gt 0 ]; then + local mf=$sysfs/device/model + local vf=$sysfs/device/vendor + local model="UNKNOWN" + local vendor="UNKNOWN" + + test -f $mf && model=$(cat $mf | sed -e 's/ *$//') + test -f $vf && vendor=$(cat $vf | sed -e 's/ *$//') + + echo "$dev: $sz; $vendor [$model]" + else + echo "$dev: $sz" fi + if [ $Partition -gt 0 ]; then + [ -n "$Fdisk" ] && $Fdisk -l $dev + fi - sz=`cat $sys/size` - if [ $sz -gt 0 ]; then - echo $dev + return 0 +} + +usage() { + + cat < 0: z = MM_CHUNKSIZE if n > MM_CHUNKSIZE else n try: mm = mmap.mmap(fdn, z, access=mmap.ACCESS_READ, offset=off) yield mm, z - except Exception, ex: + except Exception as ex: ss = "can't mmap %d bytes of %s: %s" % (z, fn, str(ex)) raise Exception(ss) mm.close() @@ -134,7 +134,7 @@ def mmap_gen(fn): def cksum_slow(filename): m = md5sum() - sz = 0L + sz = 0 for mm, n in mmap_gen(filename): buf = mm.read(n) @@ -145,8 +145,8 @@ def cksum_slow(filename): return b def cksum_quick(filename): - v = 0L - sz = 0L + v = 0 + sz = 0 for mm, n in mmap_gen(filename): buf = mm.read(n) v = zlib.adler32(buf, v) @@ -184,35 +184,35 @@ class shell: """Abstraction to print shell commands to remove dups""" def __init__(self): - self.tot = 0L - print "#! /bin/sh\n" + self.tot = 0 + print("#! /bin/sh\n") def dups(self, k, keep, rm): waste = keep.size * len(rm) self.tot += waste s = '\n'.join([ "rm -f '%s'" % x.fname for x in rm ]) - print "# %s: %s, saving %s\n#rm -f '%s'\n%s\n" % \ - (k, human(keep.size), human(waste), keep.fname, s) + print("# %s: %s, saving %s\n#rm -f '%s'\n%s\n" % \ + (k, human(keep.size), human(waste), keep.fname, s)) def finish(self): - print "# %s saved" % human(self.tot) + print("# %s saved" % human(self.tot)) class plain: """Abstraction to simply print the duplicated files""" def __init__(self): - self.tot = 0L - print "Report of duplicate files\n" + self.tot = 0 + print("Report of duplicate files\n") def dups(self, k, keep, rm): waste = keep.size * len(rm) self.tot += waste s = '\n\t'.join( [x.fname for x in rm ]) - print "%s: %s, wasted %s\n\tKEEP %s\n%s" % \ - (k, human(keep.size), human(waste), keep.fname, s) + print("%s: %s, wasted %s\n\tKEEP %s\n%s" % \ + (k, human(keep.size), human(waste), keep.fname, s)) def finish(self): - print "\n%s total wasted space" % human(self.tot) + print("\n%s total wasted space" % human(self.tot)) @@ -234,7 +234,7 @@ db = cksum_db(cksum_quick) zargs = [] for d in args: if not isdir(d): - print >>sys.stderr, "Skipping non-directory %s" % d + print("Skipping non-directory %s" % d, file=sys.stderr) continue descend(db, d) diff --git a/hexlify b/hexlify index 10aaa1d..9ba9233 100755 --- a/hexlify +++ b/hexlify @@ -8,32 +8,93 @@ # License: GPLv2 # -import os, sys, base64 +import os, sys, base64, argparse from os.path import basename, dirname, abspath, normpath, join from binascii import hexlify -from optparse import Option, OptionParser, OptionValueError from functools import partial Z = basename(sys.argv[0]) -def die(fmt, *args): - warn(fmt, args) - exit(1) +def main(): + usage = """%(Z)s - Flexible hex|base64 data dumper """ % { 'Z': Z } + + pp = argparse.ArgumentParser(description=usage) + pp.add_argument('-L', '--line-length', dest="linelen", default=0, + type=int, metavar="N", + help="Break output line into 'N' character chunks [%(default)s]") + pp.add_argument('-n', '--Nbytes', dest="N", default=0, + type=int, metavar="N", + help="Only read N bytes of input data (0 implies until EOF) [%(default)s]") + pp.add_argument("-o", "--outfile", dest="outfile", default=None, + metavar="F", type=str, + help="write output to file F [STDOUT]") + + g = pp.add_mutually_exclusive_group() + g.add_argument('-b', '--base64', dest='b64', action="store_true", default=False, + help="Base64 decode the input first [%(default)s]") + g.add_argument('-d', '--dump', dest='dump', action="store_true", default=False, + help="Show a hexdump style output [%(default)s]") + g.add_argument('-c', '--c-type', dest="ctype", action="store_true", default=False, + help="Generate a C like array definition [%(default)s]") + + pp.add_argument("infile", nargs="*", type=str, help="zero or more input files") + + args = pp.parse_args() + + out = sys.stdout + if args.outfile: + out = open(args.outfile, 'w') + + barehex = partial(hexlate, N=args.N, L=args.linelen, hexer=hexlify, pref='', suff='\n', lf="\n") + chex = partial(hexlate, N=args.N, L=args.linelen, hexer=chexlify, pref="{\n", suff="}\n", lf=",\n") + dumper = partial(dumpify, N=args.N, L=args.linelen) + + if args.ctype: + hexify = chex + elif args.dump: + hexify = dumper + else: + hexify = barehex -def warn(fmt, *args): - sfmt = "%s: %s" % (Z, fmt) - if args: sfmt = sfmt % args + if len(args.infile) > 0: + for fd, fn in argv2fd(args.infile): + hexify(fd, out, args.b64) + else: + hexify(sys.stdin, out, args.b64) - if not sfmt.endswith('\n'): sfmt += '\n' +def argv2fd(argv): + """Generator that yields open fd's from names in argv""" + if len(argv) == 0: + return - sys.stdout.flush() - sys.stderr.write(sfmt) - sys.stderr.flush() + for fn in argv: + fd = open(fn, 'rb') + yield fd, fn + fd.close() def chexlify(b): """like binascii.hexlify() except writes C like bytes""" return ', '.join(["%#2.2x" % ord(x) for x in b]) +def dumpify(infd, outfd, ign, N, L): + """Call hexdump..""" + + if L == 0: L = 16 + if N > 0: + b = infd.read(N) + if not b: + return + x = hexdump(b, stride=L) + outfd.write(x) + else: + while True: + b = infd.read(65536) + if not b: break + x = hexdump(b, stride=L) + outfd.write(x) + + outfd.write('\n') + def hexlate(infd, outfd, b64, N, L, hexer, pref, suff, lf): """Transcribe bytes from 'infd' to 'outfd' in hex. Breakup lines into 'L' bytes. If 'N' is specified, read exactly 'N' bytes from @@ -73,54 +134,34 @@ def hexlate(infd, outfd, b64, N, L, hexer, pref, suff, lf): outfd.write(suff) +def hexdump(src, stride=16, sep='.'): + chrtab = [(len(repr(chr(x))) == 3) and chr(x) or sep for x in range(256)] + lines = [] + for c in xrange(0, len(src), stride): + chars = src[c:c+stride] + hexb = ' '.join(["%02x" % ord(x) for x in chars]) + if len(hexb) > 24: + hexb = "%s %s" % (hexb[:24], hexb[24:]) + pr = ''.join(["%s" % chrtab[ord(x)] for x in chars]) + lines.append("%08x: %-*s |%s|" % (c, stride*3, hexb, pr)) -usage = """%(Z)s [options] [infile ..] [outfile] - -%(Z)s - Convert input stream into hex and optionally break it into -multiple output lines. - -""" % { 'Z': Z } + return '\n'.join(lines) -pp = OptionParser(usage=usage) -pp.add_option('-L', '--line-length', dest="linelen", action="store", - default=0, type="int", metavar="N", - help="Break output line into 'N' character chunks [%default]") -pp.add_option('-n', '--Nbytes', dest="N", action="store", - default=0, type="int", metavar="N", - help="Only read N bytes of input data (0 implies until EOF) [%default]") -pp.add_option('-b', '--base64', dest='b64', action="store_true", default=False, - help="Base64 decode the input first [%default]") -pp.add_option('-c', '--c-type', dest="ctype", action="store_true", default=False, - help="Generate a C like array definition [%default]") - -opt, args = pp.parse_args() - -out = sys.stdout -if len(args) > 1: - outf = args[-1] - args = args[:-1] - try: - out = open(outf, 'w') - except Exception, ex: - die("Can't create output file '%s': %s", outf, ex) +def die(fmt, *args): + warn(fmt, args) + exit(1) -barehex = partial(hexlate, N=opt.N, L=opt.linelen, hexer=hexlify, pref='', suff='\n', lf="\n") -chex = partial(hexlate, N=opt.N, L=opt.linelen, hexer=chexlify, pref="{\n", suff="}\n", lf=",\n") +def warn(fmt, *args): + sfmt = "%s: %s" % (Z, fmt) + if args: sfmt = sfmt % args -hexify = chex if opt.ctype else barehex + if not sfmt.endswith('\n'): sfmt += '\n' -if len(args) > 0: - for fn in args: - try: - infd = open(fn, 'rb') - except Exception, ex: - warn("Can't read file '%s': %s", fn, ex) - continue + sys.stdout.flush() + sys.stderr.write(sfmt) + sys.stderr.flush() - hexify(infd, out, opt.b64) - infd.close() -else: - hexify(sys.stdin, out, opt.b64) +main() # EOF diff --git a/kern-build b/kern-build index 0e1bb51..78b73ca 100755 --- a/kern-build +++ b/kern-build @@ -47,10 +47,10 @@ modules, it will be placed in one of two places: def error(doex, fmt, *args): sfmt = "%s: %s" % (Z, fmt) if args: - str = sfmt % args + s = sfmt % args else: - str = sfmt - print >>sys.stderr, str + s = sfmt + print(s, file=sys.stderr) if doex > 0: sys.exit(doex) @@ -113,16 +113,17 @@ def abbrev(wordlist): def grok_kver(objdir): """Grok kernel version from objdir""" - hdrdir = [ join(objdir, 'include', 'linux'), + hdrdir = [ join(objdir, 'include', 'generated'), + join(objdir, 'include', 'linux'), ] rel = None for e in ['version.h', 'utsrelease.h']: for d in hdrdir: f = join(d, e) try: - fd = file(f, 'r') - except: + fd = open(f, 'r') + except Exception as ex: continue for line in fd: @@ -144,7 +145,7 @@ def run_program(args, use_fakeroot=False): else: cmd = args - print ' '.join(cmd) + print(' '.join(cmd)) sys.stdout.flush() r = os.spawnvp(os.P_WAIT, cmd[0], cmd) if r != 0: @@ -165,9 +166,9 @@ def add_tarobj(tf, src): ti = tf.gettarinfo(src) ti.uid = 0 ti.gid = 0 - ti.mode = 0644 + ti.mode = 0o644 if os.path.isdir(src): - ti.mode |= 0111 + ti.mode |= 0o111 ti.uname = 'root' ti.gname = 'root' tf.addfile(ti) @@ -226,6 +227,7 @@ if opt.config is not None and not os.path.isfile(opt.config): Arch_aliases = { 'x86': 'x86', 'i386': 'x86', + 'amd64': 'x86', 'uml': 'um', } @@ -247,12 +249,13 @@ else: arch_suffix = "" # Find a canonical name for arch -canonarch = Arch_aliases.get(arch, arch) +origarch = arch +arch = Arch_aliases.get(arch, arch) # Make an abbreviation table for all archs present arch_ab = abbrev(os.listdir('./arch')) -if canonarch not in arch_ab: +if arch not in arch_ab: error(1, "Unknown architecture '%s'", arch) @@ -261,12 +264,12 @@ objdir = join(pwd, opt.objdir, 'obj-%s%s' % (arch, arch_suffix)) objdir = realpath(normpath(objdir)) if not os.path.isdir(objdir): - os.mkdir(objdir, 0755) + os.mkdir(objdir, 0o755) # If config file is specified, use it. defconfig = join(objdir, '.config') if not os.path.isfile(defconfig) and opt.config is not None: - print "Using config file '%s'" % opt.config + print("Using config file '%s'" % opt.config) copyfile(opt.config, defconfig) make_args = [ 'make', @@ -284,7 +287,11 @@ if not opt.tarfile: kver = grok_kver(objdir) # Substitution dict for the files list below and other places -subst = {'kver': kver, 'arch': arch, 'suffix': arch_suffix} +subst = {'kver': kver, + 'arch': arch, + 'origarch': origarch, + 'suffix': arch_suffix, + } # List of kernel output files that are needed when we build the @@ -296,19 +303,19 @@ files = { Arch_kimage[arch]: 'vmlinuz-%(kver)s', } -tarname = 'kern_%(arch)s%(suffix)s-%(kver)s.tar' % subst -tardir = os.environ.get('HOME', '.') -tarname = abspath(join(tardir, 'kernel-images', arch, tarname)) +tarname = 'kern_%(origarch)s%(suffix)s-%(kver)s.tar' % subst +tardir = os.getcwd() +tarname = abspath(join(tardir, tarname)) bztarname = tarname + '.bz2' -print "Making tarball %s .." % bztarname +print("Making tarball %s .." % bztarname) unlink_if(tarname, bztarname) # First, create a temp dir tmpdir = abspath('./tmp/kinst%d' % random.randint(1000,10000000)) kinst = join(tmpdir, 'boot') -os.makedirs(kinst, 0755) +os.makedirs(kinst, 0o755) for s, d in files.items(): src = join(objdir, s) dst = join(kinst, d % subst) @@ -330,7 +337,7 @@ run_program(depmod, True) dn = dirname(tarname) if not os.path.isdir(dn): #print "mkdir -p %s" % dn - os.makedirs(dn, 0755) + os.makedirs(dn, 0o755) tar = ['tar', 'cf', tarname, '.' ] bzip2 = ['bzip2', '-9', tarname ] diff --git a/kill.py b/kill.py index 8580849..89929fa 100755 --- a/kill.py +++ b/kill.py @@ -6,11 +6,15 @@ # License: GPLv2 # import sys, os, os.path, signal, re +from os.path import basename -Z = os.path.basename(sys.argv[0]) +Z = basename(sys.argv[0]) Verbose = False Dry_run = False Use_regex = False +Ignore_case = False +# Raw character input +Getch = None Usage = """Kill or Terminate a process. @@ -36,62 +40,34 @@ Options: -n, --dry-run Do not kill any process, only show what will be done [False] -r, --regex For processes that are named, use regex matching [False] + -i, --ignore-case Ignore case when matching names [False] -h, --help Show this help message and quit """ % (Z, Z, Z, Z, Z, Z) +def main(argv): + global Z, Verbose, Dry_run + procs, errs = parse_args(argv[1:]) -def error(doex, fmt, *args): - """Show error message and die if doex > 0""" - sfmt = "%s: %s" % (Z, fmt) - if args: - sfmt = sfmt % args - - if not sfmt.endswith('\n'): - sfmt += '\n' - - sys.stdout.flush() - sys.stderr.write(sfmt) - sys.stderr.flush() - if doex > 0: - sys.exit(doex) - - -# Raw character input -Getch = None -if sys.platform in ('win32', 'win64', 'cygwin'): - import msvcrt - - class _GetchWindows: - def __init__(self): - pass - - def __call__(self): - return msvcrt.getch() - - - Getch = _GetchWindows() - -else: - import tty, termios - - class _GetchUnix: - def __init__(self): - pass - - def __call__(self): - fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(fd) - try: - tty.setraw(sys.stdin.fileno()) - ch = sys.stdin.read(1) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - return ch + for k, v in procs.items(): + # k is the signal number, v is a list of process objects + if Dry_run: + x = '\n'.join(['kill -%d %7d %s' % (k, z.pid, z.name) for z in v]) + print(x) + continue - Getch = _GetchUnix() + for p in v: + x = ' '.join(p.command) + s = "kill -%d %7d '%s'? [y/N]? " % (k, p.pid, x) + r = prompt(s) + if r < 0: + sys.stdout.write('\n') + sys.exit(0) + if r > 0: + os.kill(p.pid, k) + sys.stdout.write('\n') class process(object): @@ -102,18 +78,8 @@ def __init__(self, **kwargs): self.parent = None self.children = [] - - def kill(self, sig=signal.SIGTERM): - os.kill(self.pid, sig) - - def killchildren(self, sig=signal.SIGTERM): - pass - - def killgroup(self, sig=signal.SIGTERM): - pass - def __str__(self): - return "%s(%d)" % (self.command, self.pid) + return "%d: %s" % (self.pid, self.name) def ps_output(pscmd, n): """Open the ps command 'ps' and yield each line as a tuple. @@ -123,57 +89,58 @@ def ps_output(pscmd, n): l = fd.readline() # first line has the titles for l in fd: l = l.strip() - v = l.split() - if len(v) > n: - x = v[n:] - v = v[:n] - v.append(' '.join(x)) - - #print '|'.join(v) + v = l.split(None, n-1) + #print "%d: %s" % (len(v), v) yield v fd.close() def _common_darwin_linux_pslist(psfmt): - """Common processing block for Darwin & Linux for grokking "ps -auxw" output.""" + """Common processing block for Darwin & Linux for grokking "ps -axww" output.""" - plist = [] + plist = {} mypid = os.getpid() for v in ps_output(psfmt, 10): - cmd = v[10] - d = {'user': v[0], - 'uid': int(v[1]), - 'pid': int(v[2]), - 'gid': int(v[3]), - 'pgid': int(v[4]), - 'ppid': int(v[5]), - 'ruser': v[6], - 'ruid': int(v[7]), - 'rgid': int(v[8]), - 'state': v[9], - 'command': cmd, - 'name': os.path.basename(cmd), - } - - if mypid != d['pid']: + d = process(user=v[0], + uid=int(v[1]), + pid=int(v[2]), + gid=int(v[3]), + pgid=int(v[4]), + ppid=int(v[5]), + ruser=v[6], + ruid=int(v[7]), + rgid=int(v[8]), + command=[v[9]], # this is a list! + ) + + d.name = basename(d.command[0]) + + + if mypid != d.pid: #print "%s - %d" % (d['name'], d['pid']) - plist.append(d) + plist[d.pid] = d return plist +# BSD, Linux ps commands are all same. +Psfmt = "ps axww -o 'user,uid,pid,gid,pgid,ppid,ruser,ruid,rgid,comm'" + def Darwin_pslist(): """PS List for Darwin returned as an array of dicts""" - psfmt = "ps -axww -o 'user,uid,pid,gid,pgid,ppid,ruser,ruid,rgid,state,comm'" - return _common_darwin_linux_pslist(psfmt) + return _common_darwin_linux_pslist(Psfmt) + +def OpenBSD6_pslist(): + """PS List for OpenBSD returned as an array of dicts""" + + return _common_darwin_linux_pslist(Psfmt) def Linux_pslist(): """PS List for Darwin returned as an array of dicts""" - psfmt = "ps axww -o 'user,uid,pid,gid,pgid,ppid,ruser,ruid,rgid,state,args'" - return _common_darwin_linux_pslist(psfmt) + return _common_darwin_linux_pslist(Psfmt) def cygwin_pslist(): @@ -181,24 +148,24 @@ def cygwin_pslist(): """ psfmt = "ps -W -l" - plist = [] + plist = {} mypid = os.getpid() for v in ps_output(psfmt, 7): cmd = v[6] - d = {'pid': int(v[0]), - 'ppid': int(v[1]), - 'pgid': int(v[2]), - 'winpid': int(v[3]), - 'uid': int(v[5]), - 'gid': 0, - 'ruid': 0, - 'rgid': 0, - 'state': 0, - 'command': cmd, - 'name': os.path.basename(cmd), - } - if mypid != d['pid']: - plist.append(d) + d = bundle(pid=int(v[0]), + ppid=int(v[1]), + pgid=int(v[2]), + winpid=int(v[3]), + uid=int(v[5]), + gid=0, + ruid=0, + rgid=0, + state=0, + command=cmd, + name=os.path.basename(cmd) + ) + if mypid != d.pid: + plist[d.pid] = d return plist @@ -214,8 +181,10 @@ class process_list(object): 'Darwin': Darwin_pslist, 'darwin': Darwin_pslist, 'Linux': Linux_pslist, + 'linux': Linux_pslist, 'linux2': Linux_pslist, 'cygwin': cygwin_pslist, + 'openbsd6': OpenBSD6_pslist, 'CYGWIN_NT-5.1': cygwin_pslist, } @@ -226,25 +195,22 @@ def __init__(self): #uname = os.uname()[0] uname = sys.platform psgrok = self.PS.get(uname, None) - assert psgrok is not None, "Can't find ps(1) specification for '%s'" % uname - - plist = psgrok() - - for d in plist: - d['plist'] = self - p = process(**d) + if psgrok is None: + error(1, "don't know how to parse ps(1) on %s", uname) - self.by_pid[p.pid] = p - self.by_name.setdefault(p.name, []).append(p) + self.by_pid = psgrok() # Create reverse mappings of parents <-> children for p in self.by_pid.values(): + self.by_name.setdefault(p.name, []).append(p) parent = self.by_pid.get(p.ppid, None) if not parent: continue parent.children.append(p) p.parent = parent + #print str(p) + def pid(self, p): @@ -262,11 +228,13 @@ def pgid(self, pgid): return x def named(self, name): - """Return all processes with the given name in a case insensitive way""" - pn = name.lower() + """Return all processes with the given name""" + global Ignore_case + + pn = name.lower() if Ignore_case else name rr = [] for nm, pv in self.by_name.items(): - ll = nm.lower() + ll = nm.lower() if Ignore_case else nm if ll == pn or ll.startswith(pn): rr += pv # Flatten and append @@ -275,23 +243,19 @@ def named(self, name): def match_rx(self, pat): """Return list of processes matching the regex 'pat'""" + global Ignore_case ret = [] - p2 = re.escape(pat) - rx = re.compile(p2, re.IGNORECASE) + rx = re.compile(pat, re.IGNORECASE if Ignore_case else 0) for pid, proc in self.by_pid.items(): #print proc - if rx.search(proc.command): + if rx.search(proc.name): ret.append(proc) return ret -def show_help(): - print Usage - - -def map_name(a, proclist): - """Given a name or pid, map it into a process object or a list + def find_all(self, a): + """Given a name or pid 'a', map it into a process object or a list of process objects. Note that a given 'name' can map to more than one process @@ -304,29 +268,33 @@ def map_name(a, proclist): pid = int(a) if pid < 0: pid = -pid - procs.append(proclist.pgid(pid)) + procs.append += self.pgid(pid) else: - p = proclist.pid(pid) + p = self.pid(pid) if p: procs.append(p) else: error(0, "Can't find pid %s", a) retval += 1 - except ValueError, e: + except ValueError as e: if Use_regex: - p = proclist.match_rx(a) + p = self.match_rx(a) else: - p = proclist.named(a) + p = self.named(a) if len(p) == 0: - retval += 1 error(0, "Can't find Process '%s'", a) + retval += 1 else: procs += p return procs, retval +def show_help(): + print(Usage) + + def parse_args(argv): """Clever command line parser that understands signal names or @@ -335,18 +303,20 @@ def parse_args(argv): It builds a system specific list of signals and their corresponding names.""" - global Verbose, Dry_run, Use_regex + global Verbose, Dry_run, Use_regex, Ignore_case # First build a mapping of signal names to signum. Also turn it # into easily testable commandline option. sigs = {} for d in dir(signal): - if d.startswith('SIG'): + if d.startswith('SIG') and not d.startswith('SIG_'): nm = d[3:] - num = eval('signal.%s' % d) + num = getattr(signal, d) + sigs[d] = num sigs[nm] = num sigs[num] = num sigs['-%s' % nm] = num + sigs['-%s' % d] = num sigs['-%d' % num] = num @@ -368,6 +338,10 @@ def parse_args(argv): Use_regex = True continue + if a == "-i" or a == "--ignore-case" or a == "--ignore": + Ignore_case = True + continue + if a.startswith('-'): sig = sigs.get(a, None) if sig is None: @@ -375,7 +349,7 @@ def parse_args(argv): # happens to be the same _value_ as a signal! error(1, "Unknown option '%s'", a) else: - killable, retval = map_name(a, proclist) + killable, retval = proclist.find_all(a) if retval == 0: x = procs.setdefault(sig, []) x += killable @@ -398,29 +372,59 @@ def prompt(s): r = 1 elif v == 'q' or v == 'Q': r = -1 - sys.stdout.write('\n') return r -def main(argv): - global Z, Verbose, Dry_run +def error(doex, fmt, *args): + """Show error message and die if doex > 0""" + sfmt = "%s: %s" % (Z, fmt) + if args: + sfmt = sfmt % args + + if not sfmt.endswith('\n'): + sfmt += '\n' + + sys.stdout.flush() + sys.stderr.write(sfmt) + sys.stderr.flush() + if doex > 0: + sys.exit(doex) - procs, errs = parse_args(argv[1:]) - for k, v in procs.items(): - # k is the signal number, v is a list of process objects - if not Dry_run: - for p in v: - s = "kill -%d %7d '%s'? [y/N]? " % (k, p.pid, p.command) - r = prompt(s) - if r > 0: - os.kill(p.pid, k) - elif r < 0: - sys.exit(0) +# global setup before main() +if sys.platform in ('win32', 'win64', 'cygwin'): + import msvcrt + + class _GetchWindows: + def __init__(self): + pass + + def __call__(self): + return msvcrt.getch() + + + Getch = _GetchWindows() + else: - x = '\n'.join(['kill -%d %7d %s' % (k, z.pid, z.command) for z in v]) - print x + import tty, termios + + class _GetchUnix: + def __init__(self): + pass + + def __call__(self): + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(sys.stdin.fileno()) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + + + Getch = _GetchUnix() main(sys.argv) diff --git a/linktest.py b/linktest.py index 1846ce2..d6abf4d 100755 --- a/linktest.py +++ b/linktest.py @@ -35,24 +35,16 @@ def verbose(fmt, *args): if not Verbose: return - if args: - str = fmt % args - else: - str = fmt - - print >>sys.stderr, str + s = (fmt % args) if args else fmt + print(s, file=sys.stderr) sys.stderr.flush() def error(doex, fmt, *args): """Show error message and die if doex > 0""" sfmt = "%s: %s" % (Z, fmt) - if args: - str = sfmt % args - else: - str = sfmt - - print >>sys.stderr, str + s = (fmt % args) if args else fmt + print(s, file=sys.stderr) if doex > 0: sys.exit(doex) @@ -252,8 +244,8 @@ def link_test(cmdobj, iface): if fields['link'] != 'yes': result = "NOPE" - print "%s %s link=%s speed=%s duplex=%s" % (result, iface, fields['link'], - fields['speed'], fields['duplex']) + print("%s %s link=%s speed=%s duplex=%s" % (result, iface, fields['link'], + fields['speed'], fields['duplex'])) # -- main() -- @@ -276,7 +268,7 @@ def link_test(cmdobj, iface): cmdobj = local_command() for a in args: if not os.path.isdir('/sys/class/net/%s' % a): - print "ERR %s No such interface" % a + print("ERR %s No such interface" % a) continue link_test(cmdobj, a) diff --git a/mk-raw-vmware.py b/mk-raw-vmware.py index 6ba672b..4c31e7b 100755 --- a/mk-raw-vmware.py +++ b/mk-raw-vmware.py @@ -11,7 +11,7 @@ def make_uuid(filename): if len(sys.argv) < 2: - print "usage: %s rawdiskimagefilename [vmdk name]" % sys.argv[0] + print("usage: %s rawdiskimagefilename [vmdk name]" % sys.argv[0]) sys.exit(1) filename = abspath(normpath(sys.argv[1])) @@ -20,11 +20,11 @@ def make_uuid(filename): vmdkfilename = sys.argv[2] if not os.path.exists(filename): - print "File not found: %s" % filename + print("File not found: %s" % filename) sys.exit(1) r = 0x1a1e708919272998fadcb2aa11faf63fL -r += random.randint(1L << 32, 1L << 40) +r += random.randint(1 << 32, 1 << 40) vmdk = file(vmdkfilename, 'w') diff --git a/openbsd-randmac.sh b/openbsd-randmac.sh index 2cb4150..01d7376 100755 --- a/openbsd-randmac.sh +++ b/openbsd-randmac.sh @@ -43,7 +43,7 @@ _hexrand() { _randmac() { # These are VMware, Xen and Parallels OUIs set -A vendors "00:05:69" "00:0c:29" "00:1c:14" "00:50:56" \ - "00:1c:42" "00:16:3e" + "00:1c:42" "00:16:3e" "00:bb:3a" "e0:cb:1d" typeset n=${#vendors[@]} typeset rr=$(_rand4) diff --git a/osx-randmac.sh b/osx-randmac.sh index d858b47..3ba2863 100755 --- a/osx-randmac.sh +++ b/osx-randmac.sh @@ -182,7 +182,7 @@ function _randmac { # These are VMware, Xen and Parallels OUIs typeset -a vendors=("00:05:69" "00:0c:29" "00:1c:14" "00:50:56" \ - "00:1c:42" "00:16:3e") + "00:1c:42" "00:16:3e" "00:bb:3a" "e0:cb:1d") typeset -a rand=($randstr) # Number of vendors (length of array) @@ -203,7 +203,7 @@ function _randmac { function update_mac { typeset iface=$1 - typeset mac=$(_randmac) + typeset mac=$2 # Updating MAC requires us to turn on the WiFi iface @@ -310,11 +310,16 @@ else test $? -eq 0 || exit 1 fi +mac=$2 +if [ -z "$mac" ]; then + mac=$(_randmac) +fi + r=0 case $action in start|update) - update_mac $iface + update_mac $iface $mac r=$? ;; diff --git a/pingsubnet.py b/pingsubnet.py index d4a311b..ce477ed 100755 --- a/pingsubnet.py +++ b/pingsubnet.py @@ -39,13 +39,15 @@ import socket, string import argparse from binascii import hexlify +from datetime import datetime +from operator import attrgetter # total size of data (payload) ICMP_DATA_SIZE = 56 ICMP_TYPE = 8 ICMP_CODE = 0 -PID = 0xffff & os.getpid() +IPPROTO_ICMP = 1 Z = os.path.basename(sys.argv[0]) __doc__ = """%s - Ping all the hosts in one or more subnets and return the result. @@ -53,48 +55,127 @@ Usage: %s [options] SUBNET [SUBNET..] """ % (Z, Z) -def die(fmt, *args): - """Exit if running standalone, else raise an exception - """ - warn(fmt, *args) - sys.exit(1) +Debug = 0 -def warn(fmt, *args): - sfmt = "%s: %s" % (Z, fmt) - if len(args) > 0: - sfmt = sfmt % args +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("-w", "--wait", dest='wait', action="store", type=float, + default=1.0, metavar="T", + help="Wait T seconds to conclude a host dead [1.0]") + parser.add_argument("-n", "--max-packets", dest='maxpkt', action="store", + type=int, default=3, metavar="N", + help="Send 'N' ICMP_ECHO packets [%(default)d]") + parser.add_argument("-s", "--size", dest='size', action="store", + type=int, default=ICMP_DATA_SIZE, metavar="N", + help="Send 'N' bytes of data in the ping packet [%(default)d]") + parser.add_argument("-b", "--brief", dest='brief', action="store_true", + default=False, + help="Show only IP addresses [False]") - if not sfmt.endswith('\n'): - sfmt += '\n' - sys.stderr.write(sfmt) - sys.stderr.flush() + g = parser.add_mutually_exclusive_group() + g.add_argument("--all", dest='all', action="store_true", + help="Show all hosts and responses [False]") + g.add_argument("-d", "--dead", dest='dead', action="store_true", + help="Show all hosts that are non-response [False]") + g.add_argument("-a", "--alive", dest='alive', action="store_true", + help="Show all hosts that are responsive [True]") -def _in_cksum(packet): - """THE RFC792 states: 'The 16 bit one's complement of - the one's complement sum of all 16 bit words in the header.' + parser.add_argument("subnets", nargs="+", type=ipv4, help="SUBNET [SUBNET..]") - Generates a checksum of a (ICMP) packet. Based on in_chksum found - in ping.c on FreeBSD. - """ + args = parser.parse_args() - # add byte if not dividable by 2 - if len(packet) & 1: - packet = packet + '\0' + h = {} + maxpkt = args.maxpkt - # split into 16-bit word and insert into a binary array - words = array.array('h', packet) - s = 0 + size = 0 + for sn in args.subnets: + for a in sn: + s = a.addrstr() + h[s] = ping(a, maxpkt) - # perform ones complement arithmetic on 16-bit words - for word in words: - s += (word & 0xffff) - hi = s >> 16 - lo = s & 0xffff - s = hi + lo - s = s + (s >> 16) + simulping(h, args) - return (~s) & 0xffff # return ones complement + if args.all: + z = h.values() + elif args.dead: + z = deadhosts(h) + else: + z = livehosts(h) + + zs = sorted(z, key=attrgetter('ipaddr')) + if args.brief: + zp = ( v.addr for v in zs ) + else: + zp = ( "%15s: %s" % (v.addr, v.summary()) for v in zs ) + + print('\n'.join(zp)) + +def simulping(h, args): + """Ping concurrently""" + + fd = sockv4() + timeout = args.wait + size = args.maxpkt + rem = pending(h) + + debug("%d total hosts ..", len(rem)) + + # Prime the pump + for a, p in h.items(): + + # Send the first packet to each of the hosts + pkt = p.next_packet() + try: + fd.sendto(pkt, (a, 1)) + except Exception as ex: + debug("tx %s: %s", a, str(ex)) + pass + + # Now, wait for things to come back + while len(rem) > 0: + debug("%d pending .. \r" % len(rem)) + + rfd = [] + while True: + rfd, wfd, xfd = select.select([fd], [], [], timeout) + break + + # Notify everyone that we lost a bloody packet. + if not rfd: + debug("Timeout; rem %d", len(rem)) + z = [] + for v in rem: + v.lostpkt() + pkt = v.next_packet() + if pkt: + try: + z.append(v) + fd.sendto(pkt, (v.addr, 1)) + except Exception as ex: + debug("tx2 %s: %s", v.addr, str(ex)) + pass + + # Restart the wait loop + rem = z + continue + + endtime = now() + pkt, tup = fd.recvfrom(size+48) + addr = tup[0] + debug("%s: RX %d bytes", addr, len(pkt)) + v = h.get(addr, None) + if not v: + #warn("%s: Wha?", addr) + continue + + v.reply_in(pkt, endtime) + + pkt = v.next_packet() + if pkt: + fd.sendto(pkt, (addr, 1)) + + #rem = pending(h) @@ -104,13 +185,13 @@ def sockv4(): try: sock = socket.socket(socket.AF_INET, socket.SOCK_RAW, \ socket.getprotobyname("icmp")) - except socket.error, e: + except socket.error as e: die("You must be root (%s uses raw sockets)" % os.path.basename(sys.argv[0])) return sock -class ipv4(object): +class ipv4: """A Useful IPv4 address class. IPv4 Addresses can be constructed like so: @@ -131,18 +212,20 @@ class ipv4(object): """ + _max = 32 + def _parse(self, st="0.0.0.0"): """Parse an ipv4 address string""" - if type(st) == long: - if st > 0xffffffffL: + if type(st) == int: + if st > 0xffffffff: raise ValueError, "Malformed IP address '%l'" % st return st & 0xffffffffL try: - x = long(st) - if x > 0xffffffffL: + x = int(st) + if x > 0xffffffff: raise ValueError, "Malformed IP address '%s'" % st return x & 0xffffffffL except: @@ -153,74 +236,54 @@ def _parse(self, st="0.0.0.0"): raise ValueError, "Malformed IP address '%s'" % st try: - vi = map(long, v) + vi = map(int, v) except: raise ValueError, "Malformed IP address '%s'" % st + z = 0 for x in vi: if x > 255 or x < 0: raise ValueError, "Malformed IP address '%s'" % st + z = (z << 8) | x - x = (vi[0] << 24) | (vi[1] << 16) | (vi[2] << 8) | vi[3] - return x & 0xffffffffL + return z & 0xffffffff def _parse_mask(self, st): """Intelligently grok a netmask""" - if type(st) == long or type(st) == int: + if type(st) == int: if st > self._max: return st - else: - return 0xffffffffL & (((2L << (st - 1)) - 1) << (32 - st)) + return prefix_to_ipv4(st) if st.find('.') < 0: try: - v = long(st) + v = int(st) except: raise ValueError, "Malformed netmask '%s'" % st if v > self._max: raise ValueError, "Too many bits in netmask '%s'" % st - return 0xffffffffL & (((2L << (v - 1)) - 1) << (32 - v)) + return prefix_to_ipv4(v) else: return self._parse(st) @classmethod def tostr(cls, v): - #v = a._addr & 0xffffffffL; - v0 = v & 0xff - v1 = (v >> 8) & 0xff - v2 = (v >> 16) & 0xff - v3 = (v >> 24) & 0xff - return "%s.%s.%s.%s" % (v3, v2, v1, v0) - - def _tostr(self, v): - """Convert 'v' to string quad""" - return ipv4.tostr(v) + return _tostr(v) def _masklen(self, v): """Return cidr mask - number of set bits """ - # We actually cheat and count zero bits from the left and - # subtract it from the max - nz = 0L - while v > 0: - if v & 1: - break - - nz += 1 - v >>= 1 - - return self._max - nz + return ipv4_to_prefix(v) def __init__(self, addr, mask=None): """Construct an IPv4 address""" - self._max = 32L if type(addr) != type(""): addr = repr(addr) @@ -244,60 +307,67 @@ def __init__(self, addr, mask=None): def __repr__(self): - return "%s/%d" % (self._tostr(self._addr), self._cidr) - - - def __cmp__(self, other): - """Return -1 if self < other; 0 if self == other; 1 if self > other""" - a = self._addr - b = other._addr - x = a - b - if x < 0: - y = -1 - elif x > 0: - y = +1 - else: - y = 0 - return y + return "%s/%d" % (_tostr(self._addr), self._cidr) + + def __str__(self): + return self.__repr__() + + + def __eq__(a, b): + return a._addr == b._addr + + def __lt__(a, b): + return a._addr < b._addr + def __le__(a, b): + return a._addr <= b._addr + + def __gt__(a, b): + return a._addr > b._addr + def __ge__(a, b): + return a._addr >= b._addr + + def __ne__(a, b): + return a._addr != b._addr def __hash__(self): """Return int usable as a key to dict""" return int(self._addr & 0x7fffffffL) + def __iter__(self): + """Return iterator for range represented by this CIDR""" + return _ipv4iter(self) + + def __len__(self): + """Return number of hosts spanned by this address range""" + n = self._max - self._cidr + return int(1 << n) + # Accessor methods def cidr(self): return self.__repr__() def first(self): """Return the first IP address of the range""" - net = 0xffffffffL & (self._addr & self._mask) + net = 0xffffffff & (self._addr & self._mask) return ipv4(net, self._mask) - def __iter__(self): - """Return iterator for range of addresses represented by this - class""" - return _ipv4iter(self) - def count(self): """Count number of addresses in the range""" f = self._addr - l = 0xffffffffL & (self._addr | ~self._mask) + l = 0xffffffff & (self._addr | ~self._mask) return l - f def last(self): """Return the last IP address of the range""" - l = 0xffffffffL & (self._addr | ~self._mask) + l = 0xffffffff & (self._addr | ~self._mask) return ipv4(l, self._mask) def standard(self): - return "%s/%s" % (self._tostr(self._addr), self._tostr(self._mask)) + return "%s/%s" % (_tostr(self._addr), _tostr(self._mask)) def addr(self): return self._addr - def addrstr(self): - return ipv4.tostr(self._addr) - def netmask(self): return self._mask @@ -305,16 +375,21 @@ def netmask_cidr(self): return self._masklen(self._mask) + def addrstr(self): + return _tostr(self._addr) + + def maskstr(self): + return _tostr(self._mask) + def net(self): """Return network number of this address+mask""" - return 0xffffffffL & (self._addr & self._mask) + return 0xffffffff & (self._addr & self._mask) def is_member_of(self, net): """Return true if IP is member of network 'net'""" - try: - v = net.netmask_cidr - except: + fp = getattr(net, "netmask_cidr", None) + if fp is None: net = ipv4(net) mynet = self._addr & net._mask; @@ -322,12 +397,39 @@ def is_member_of(self, net): return mynet == theirnet def network(self): - net = 0xffffffffL & (self._addr & self._mask) - return "%s/%s" % (self._tostr(net), self._tostr(self._mask)) + net = 0xffffffff & (self._addr & self._mask) + return "%s/%s" % (_tostr(net), _tostr(self._mask)) def network_cidr(self): - net = 0xffffffffL & (self._addr & self._mask) - return "%s/%d" % (self._tostr(net), self._cidr) + net = 0xffffffff & (self._addr & self._mask) + return "%s/%d" % (_tostr(net), self._cidr) + + tostr = classmethod(tostr) + + +def ipv4_to_prefix(n): + """Convert IPv4 address 'a' (in integer representation) into prefix format.""" + + if n == 0xffffffff: return 32 + if n == 0: return 0 + + for i in range(32): + if n & 1: return 32-i + n >>= 1 + + +def prefix_to_ipv4(n): + """Convert a 32-bit network prefix length into a IPv4 address""" + ones = 0xffffffff + return ones ^ (ones >> n) + +def _tostr(v): + """Convert IPv4 to dotted quad string""" + v0 = v & 0xff + v1 = (v >> 8) & 0xff + v2 = (v >> 16) & 0xff + v3 = (v >> 24) & 0xff + return "%s.%s.%s.%s" % (v3, v2, v1, v0) class _ipv4iter(object): @@ -343,7 +445,7 @@ def __iter__(self): def next(self): """Return the next address after self.cur""" - m = 0xffffffffL & (self.cur & self.mask) + m = 0xffffffff & (self.cur & self.mask) if m != self.last: raise StopIteration @@ -351,55 +453,44 @@ def next(self): self.cur += 1 return ipv4(n, self.mask) -def icmpv4(idz, size, ipv6): + +def icmpv4(pktid, seq, size, ipv6): """Constructs a ICMP echo packet of variable size """ - if size < int(struct.calcsize("d")): - die("packet size too small, must be at least %d", int(struct.calcsize("d"))) - - header = struct.pack('bbHHh', ICMP_TYPE, ICMP_CODE, 0, \ - PID, idz) - - # if size big enough, embed this payload - load = "--PING-PONG KING-KONG PING-KING-PONG-KONG!--" - - # space for time - size -= struct.calcsize("d") + if size < 16: die("packet size too small, must be at least 16") - # construct payload based on size, may be omitted :) + header = struct.pack('BBHHH', ICMP_TYPE, ICMP_CODE, 0, pktid, seq) + body = string.printable + size -= struct.calcsize("Q") rest = "" - if size > len(load): - rest = load - size -= len(load) - - # pad the rest of payload - rest += size * "X" + while size > 0: + n = size if len(body) > size else len(body) + rest += body[:n] + size -= n - # pack - data = struct.pack("d", time.time()) + rest - checksum = _in_cksum(header+data) # make checksum + ts = now() + data = struct.pack("Q", ts) + rest + checksum = _in_cksum(header+data) + header = struct.pack('BBHHH', ICMP_TYPE, ICMP_CODE, checksum, pktid, seq) - # Redo the header with the right checksum - header = struct.pack('bbHHh', ICMP_TYPE, ICMP_CODE, checksum, PID, \ - idz) - - # ping packet *with* checksum return header + data class ping(object): """Abstraction of a ping request and a response for a given host. """ - def __init__(self, addr, maxpkts, ipv6=False, size=ICMP_DATA_SIZE): + def __init__(self, addr, maxpkts, pktid=os.getpid(), ipv6=False, size=ICMP_DATA_SIZE): # size must be big enough to contain time sent if size < int(struct.calcsize("d")): die("packetsize to small, must be at least %d" % int(struct.calcsize("d"))) - self.addr = addr + self.ipaddr = addr + self.addr = addr.addrstr() + self.pktid = pktid self.ipv6 = ipv6 self.size = size self.seq = 1; - self.mint = 999 + self.mint = 999 * 1000 self.maxt = 0.0 self.avg = 0.0 self.lost = 0 @@ -419,7 +510,7 @@ def next_packet(self): if self.seq >= self.lastpkt: return None - pkt = icmpv4(self.seq, self.size, self.ipv6) + pkt = icmpv4(self.pktid, self.seq, self.size, self.ipv6) self.seq += 1 self.txpkt += 1 @@ -448,40 +539,45 @@ def reply_in(self, pkt, endtime): #rawPongHop = struct.unpack("c", pkt[7])[0] # fetch pkt header - pktHeader = pkt[0:8] - pktType, pktCode, pktChksum, pktID, pktSeqnr = \ - struct.unpack("bbHHh", pktHeader) + typ, code, cksum, pktid, seq = struct.unpack("bbHHh", pkt[0:8]) # fetch starttime from pkt starttime = struct.unpack("d", pkt[8:16])[0] # IPv4 else: - # time to live - rawPongHop = struct.unpack("s", pkt[8])[0] + # XXX We assume no IP extension headers are present. + # B B: V+IHL, DSCP+ESCN + # H: Total Len + # H: id + # H: flags + fragoff + # B B: TTL, Proto + # H: Header cksum + # I, I: Srcaddr, destaddr + pv = struct.unpack("BBHHHBBHII", pkt[0:20]) + + proto = pv[6] + if proto != IPPROTO_ICMP: return + + orig = pkt + pkt = pkt[20:36] + typ, code, cksum, pktid, seq = struct.unpack("BBHHH", pkt[0:8]) - # convert TTL from 8 bit to 16 bit integer - pktHop = int(hexlify(str(rawPongHop)), 16) + # fetch starttime from pkt + starttime = struct.unpack("Q", pkt[8:])[0] + if pktid != self.pktid: return - # fetch pkt header - pktHeader = pkt[20:28] - pktType, pktCode, pktChksum, pktID, pktSeqnr = \ - struct.unpack("bbHHh", pktHeader) - # fetch starttime from pkt - starttime = struct.unpack("d", pkt[28:36])[0] + if seq >= self.seq: return - # valid ping packet received? - #print >>sys.stderr, "%s: start=%d, rx=%d" % (self.addr, self.seq, pktSeqnr) - if pktSeqnr < self.seq: self.rxpkt += 1 - triptime = endtime - starttime # compute RRT - self.tsum += triptime # triptime for all packets (stddev) - self.tsumsq += triptime * triptime # triptime^2 for all packets (stddev) + triptime = endtime - starttime + self.tsum += triptime + self.tsumsq += triptime * triptime # compute statistic - self.maxt = max ((triptime, self.maxt)) - self.mint = min ((triptime, self.mint)) + self.maxt = max(triptime, self.maxt) + self.mint = min(triptime, self.mint) def done(self): @@ -499,11 +595,7 @@ def summary(self, verb=False): self.rxpkt = self.txpkt # compute and print som stats - # stddev computation based on ping.c from FreeBSD lost = self.txpkt - self.rxpkt - avg = self.tsum / self.rxpkt - vari = abs((self.tsumsq / self.rxpkt) - (avg * avg)) - # %-packet lost plost = 100.0 * (float(lost) / float(self.txpkt)) s1 = "%d TX, %d RX, %d%% loss" % \ @@ -511,15 +603,22 @@ def summary(self, verb=False): s2 = "" if self.rxpkt > 0: - try: + # stddev computation based on ping.c from FreeBSD + avg = (float(self.tsum) / self.rxpkt) + vari = abs((float(self.tsumsq) / self.rxpkt) - (avg * avg)) s2 = " RTT min %.3f ms avg %.3f ms max %.3f ms stddev %.3f" % \ - (self.mint*1000, avg*1000, self.maxt*1000, math.sqrt(vari)*1000) - except Exception, ex: - warn("%s: %s\n avg %.3f vari %.3f", self.addr, str(ex), avg, vari) - + (self.mint/1000.0, avg/1000.0, self.maxt/1000.0, math.sqrt(vari)/1000.0) return s1+s2 + +def now(): + """Return time in microseconds as uint64 """ + n = datetime.utcnow() + s = 0 + (((n.hour * 60) + n.minute) * 60) + n.second + s = n.microsecond + (s * 1000000) + return s + def pending(z): """Walk the dict and return pending items""" return [ v for v in z.values() if not v.done() ] @@ -531,133 +630,73 @@ def livehosts(z): return [v for v in z.values() if not v.dead() ] -def simulping(h, args): - fd = sockv4() - timeout = args.wait - size = args.maxpkt - rem = pending(h) - - #warn("%d total hosts ..", len(rem)) - - # Prime the pump - for a, p in h.items(): - - # Send the first packet to each of the hosts - pkt = p.next_packet() - try: - fd.sendto(pkt, (a, 1)) - except Exception, ex: - #warn("%s: %s", a, str(ex)) - pass - - # Now, wait for things to come back - while len(rem) > 0: - #sys.stdout.write("%d pending .. \r" % len(rem)) - #sys.stdout.flush() - #print "%d pending" % len(rem) - - rfd = [] - while True: - rfd, wfd, xfd = select.select([fd], [], [], timeout) - break - - # Notify everyone that we lost a bloody packet. - if not rfd: - #warn("Timeout; rem %d", len(rem)) - z = [] - for v in rem: - v.lostpkt() - pkt = v.next_packet() - if pkt: - try: - z.append(v) - fd.sendto(pkt, (v.addr, 1)) - except Exception, ex: - #warn("%s: %s", v.addr, str(ex)) - pass - - # Restart the wait loop - rem = z - continue - - endtime = time.time() # time packet received - pkt, tup = fd.recvfrom(size+48) - addr = tup[0] - #warn("%s: RX %d bytes", addr, len(pkt)) - #print >>sys.stderr, "RX %d bytes from %s" % (len(pkt), addr) - v = h.get(addr, None) - if not v: - #warn("%s: Wha?", addr) - continue - - v.reply_in(pkt, endtime) +def die(fmt, *args): + """Exit if running standalone, else raise an exception + """ + warn(fmt, *args) + sys.exit(1) - pkt = v.next_packet() - if pkt: - fd.sendto(pkt, (addr, 1)) +def warn(fmt, *args): + sfmt = "%s: %s" % (Z, fmt) + if len(args) > 0: + sfmt = sfmt % args - #rem = pending(h) + if not sfmt.endswith('\n'): + sfmt += '\n' + sys.stderr.write(sfmt) + sys.stderr.flush() -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("-w", "--wait", dest='wait', action="store", type=float, - default=1.0, metavar="T", - help="Wait T seconds to conclude a host dead [1.0]") - parser.add_argument("-n", "--max-packets", dest='maxpkt', action="store", - type=int, default=3, metavar="N", - help="Send 'N' ICMP_ECHO packets [%(default)d]") - parser.add_argument("-s", "--size", dest='size', action="store", - type=int, default=ICMP_DATA_SIZE, metavar="N", - help="Send 'N' bytes of data in the ping packet [%(default)d]") - parser.add_argument("-v", "--verbose", dest='verbose', action="store_true", - default=False, - help="Show verbose ping summary [False]") +def debug(fmt, *args): + global Debug + if not Debug: + return - g = parser.add_mutually_exclusive_group() - g.add_argument("--all", dest='all', action="store_true", - help="Show all hosts and responses [False]") - g.add_argument("-d", "--dead", dest='dead', action="store_true", - help="Show all hosts that are non-response [False]") - g.add_argument("-a", "--alive", dest='alive', action="store_true", - help="Show all hosts that are responsive [True]") + z = fmt % args if len(args) > 0 else fmt + if not z.endswith('\n'): z += '\n' - parser.add_argument("subnets", nargs="+", type=ipv4, help="SUBNET [SUBNET..]") + sys.stderr.write(z) + sys.stderr.flush() - args = parser.parse_args() - h = {} - maxpkt = args.maxpkt +def _in_cksum(packet): + """THE RFC792 states: 'The 16 bit one's complement of + the one's complement sum of all 16 bit words in the header.' - size = 0 - for sn in args.subnets: - for i in sn: - a = i.addrstr() - p = ping(a, maxpkt) - h[a] = p + Generates a checksum of a (ICMP) packet. Based on in_chksum found + in ping.c on FreeBSD. + """ + # add byte if not dividable by 2 + if len(packet) & 1: + packet = packet + '\0' - if args.verbose: - print "# %d total hosts" % len(h) + # split into 16-bit word and insert into a binary array + words = array.array('h', packet) + s = 0 - simulping(h, args) + # perform ones complement arithmetic on 16-bit words + for word in words: + s += (word & 0xffff) - if args.all: - z = h.values() - elif args.dead: - z = deadhosts(h) - else: - z = livehosts(h) + hi = s >> 16 + lo = s & 0xffff + s = hi + lo + s = s + (s >> 16) - # ipv4 objects can be naturally compared! - zs = sorted(z) - zp = [] - if args.verbose: - zp = ( "%15s: %s" % (v.addr, v.summary()) for v in zs ) - else: - zp = ( v.addr for v in zs ) + return (~s) & 0xffff # return ones complement - print '\n'.join(zp) - +def hexdump(src, stride=16, sep='.'): + chrtab = [(len(repr(chr(x))) == 3) and chr(x) or sep for x in range(256)] + lines = [] + for c in xrange(0, len(src), stride): + chars = src[c:c+stride] + hexb = ' '.join(["%02x" % ord(x) for x in chars]) + if len(hexb) > 24: + hexb = "%s %s" % (hexb[:24], hexb[24:]) + pr = ''.join(["%s" % chrtab[ord(x)] for x in chars]) + lines.append("%08x: %-*s |%s|" % (c, stride*3, hexb, pr)) + + return '\n'.join(lines) main() diff --git a/randmac b/randmac index e8d67f1..a145b73 100755 --- a/randmac +++ b/randmac @@ -78,8 +78,8 @@ pref = "%s:%s:%s" % (p[0:2], p[2:4], p[4:]) mac = "%s:%s" % (pref.lower(), randhex()) if opt.verbose: - print "%s - %s" % (mac, t[1]) + print("%s - %s" % (mac, t[1])) else: - print mac + print(mac) # EOF diff --git a/randpass b/randpass index 3280037..0b6f732 100755 --- a/randpass +++ b/randpass @@ -17,7 +17,7 @@ from optparse import Option, OptionParser, OptionValueError def randnum(n=4): """Return a 'n' decimal digit random number""" - l = 10L ** (n - 1) + l = 10 ** (n - 1) #print "n=%d, %ld <= x < %ld" % (n, l, l *10) v = random.randint(l, (l * 10) - 1) return "%ld" % v @@ -183,6 +183,6 @@ else: rr = ( gen() for a in range(n) ) ps = '\0' if opt.print0 else '\n' -print ps.join(rr) +print(ps.join(rr)) # EOF diff --git a/realpath b/realpath index 90f063a..16561de 100755 --- a/realpath +++ b/realpath @@ -5,4 +5,4 @@ import os.path, sys # Eliminate symbolic links and return the canonical path without # superfluous ../../..; // etc. for a in sys.argv[1:]: - print os.path.realpath(os.path.normpath(a)) + print(os.path.realpath(os.path.normpath(a))) diff --git a/rm.py b/rm.py index 8876470..0cf7751 100755 --- a/rm.py +++ b/rm.py @@ -91,14 +91,14 @@ def prompt(s): r = 1 elif v == 'q' or v == 'Q': r = -1 - sys.stdout.write('\n') + #sys.stdout.write('\n') return r def main(argv): if len(argv) == 1 or argv[1] == '-h': - print "Usage: %s FILE [FILE..]" % argv[0] + print("Usage: %s FILE [FILE..]" % argv[0]) sys.exit(1) for f in argv[1:]: @@ -108,10 +108,16 @@ def main(argv): s = "%-50s [y/N]? " % f r = prompt(s) if r < 0: + sys.stdout.write('\n') sys.exit(0) elif r > 0: + try: os.unlink(f) + except Exception as ex: + s = str(ex) + sys.stdout.write(" %s\n" % s) + sys.stdout.write('\n') main(sys.argv) diff --git a/tolower b/tolower index 1b67f46..c60770f 100755 --- a/tolower +++ b/tolower @@ -56,13 +56,13 @@ else: def dummy_rename(old, new): """Print what would happen""" - print "mv '%s' '%s'" % (old, new) + print("mv '%s' '%s'" % (old, new)) def verbose_real(fmt, *args): sfmt = fmt if args: sfmt = fmt % args - print sfmt + print(sfmt) sys.stdout.flush() def no_verbose(fmt, *args): @@ -123,8 +123,8 @@ def do_rename(a, b): try: rename(a, b) - except Exception, e: - print >>sys.stderr, "Can't rename '%s' -> '%s': %s" % (a, b, str(e)) + except Exception as e: + print("Can't rename '%s' -> '%s': %s" % (a, b, str(e)), file=sys.stderr) def lowerize(p, transform): """Convert 'p' to lowercase""" @@ -168,13 +168,13 @@ def unit_test(transform): test_vectors = ['abc.txt', 'abc DEF ghi.txt', 'ABC DEF.txt', 'Abc_Def.txt', 'ABC.TXT' ] for t in test_vectors: n = transform(t) - print "%s => %s" % (t, n) + print("%s => %s" % (t, n), file=sys.stderr) # -- main -- if len(sys.argv) < 2: - print >>sys.stderr, "Usage: %s file [...]" % sys.argv[0] + print("Usage: %s file [...]" % sys.argv[0], file=sys.stderr) sys.exit(1) rename = dummy_rename @@ -207,7 +207,7 @@ parser.add_option("-M", "--mixed-case", dest="mixed_case", action="store_true", (opt, args) = parser.parse_args() if len(args) < 1: - print >>sys.stderr, "%s: Insufficient arguments. Try '%s --help'" % (Z, Z) + print("%s: Insufficient arguments. Try '%s --help'" % (Z, Z), file=sys.stderr) sys.exit(1)