parse and decode Redis AOF file with Python

Decode and parse Redis AOF log appendonly.aof into plain text, and upload it to HDFS or load it into Hive for further processing.

RESP

Redis uses the RESP protocol (Redis Serialization Protocol), and the AOF file is the persistent record of this protocol. The RESP format of a command is as follows:

*<Number of arguments>\r\n
$<Number of bytes of the first argument/command>\r\n
<Argument 1>\r\n
...
$<Number of bytes of argument N>\r\n
<Argument N>\r\n

For example, the RESP format of the command set key:000000000007 VXK is:

*3
$3
SET
$16
key:000000000007
$3
VXK

parse AOF

#!/usr/bin/env python3
import argparse

def decode_aof(filename):
    key_no = 0
    v_sign = False
    print_sign = False
    key = ''
    val = ''
    with open(filename, 'r') as f:
        for line in f:
            if line.startswith('*'):
                key_no = 0
                v_sign = False
                if print_sign == 1:
                    print(key, '\t', val)
                    print_sign = 0
                    val = ''
            if key_no == 4:
                key = line.replace('\n', '')
                key_no += 1
                v_sign = True
                continue
            if v_sign is True and not line.startswith('$'):
                val += str(line.strip('\r\n'))
                print_sign = True
            key_no += 1
    print(key, '\t', val)


if __name__ == '__main__':
    p = argparse.ArgumentParser(description='decode tool', conflict_handler='error')
    p.add_argument('aoffile', type=str, nargs='*', help='aoffile')
    args = p.parse_args()
    if not args.aoffile:
        p.parse_args(['-h'])
    else:
        decode_aof(args.aoffile[0])

test example, appendonly.aof

$6
SELECT
$1
0
*3
$3
SET
$16
key:000000000007
$3
VXK
*3
$3
SET
$16
key:000000000000
$3
VXK
*3
$3
SET
$16
key:000000000004
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000006
$3
VXK
*3
$3
SET
$16
key:000000000004
$3
VXK
*3
$3
SET
$16
key:000000000008
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000005
$3
VXK
*3
$3
SET
$16
key:000000000006
$3
VXK
*3
$4
SADD
$5
myset
$20
element:000000000006
*3
$4
SADD
$5
myset
$20
element:000000000008
*3
$4
SADD
$5
myset
$20
element:000000000007
*3
$4
SADD
$5
myset
$20
element:000000000009
*3
$4
SADD
$5
myset
$20
element:000000000003
*3
$4
SADD
$5
myset
$20
element:000000000001
*4
$4
HSET
$6
myhash
$20
element:000000000004
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000009
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000007
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000001
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000002
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000001
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000005
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000008
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000009
$3
VXK
*4
$4
HSET
$6
myhash
$20
element:000000000004
$3
VXK
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000008
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000002
*4
$4
ZADD
$6
myzset
$12
000000000006
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000007
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000009
$20
element:000000000004
*4
$4
ZADD
$6
myzset
$12
000000000000
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000005
$20
element:000000000009
*4
$4
ZADD
$6
myzset
$12
000000000005
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000002
$20
element:000000000006
*4
$4
ZADD
$6
myzset
$12
000000000003
$20
element:000000000004

The result and output

key:000000000007   VXK
key:000000000000   VXK
key:000000000004   VXK
key:000000000005   VXK
key:000000000006   VXK
key:000000000004   VXK
key:000000000008   VXK
key:000000000005   VXK
key:000000000005   VXK
key:000000000006   VXK
myset   element:000000000006
myset   element:000000000008
myset   element:000000000007
myset   element:000000000009
myset   element:000000000003
myset   element:000000000001
myhash   element:000000000004VXK
myhash   element:000000000009VXK
myhash   element:000000000007VXK
myhash   element:000000000001VXK
myhash   element:000000000002VXK
myhash   element:000000000001VXK
myhash   element:000000000005VXK
myhash   element:000000000008VXK
myhash   element:000000000009VXK
myhash   element:000000000004VXK
myzset   000000000006element:000000000008
myzset   000000000006element:000000000002
myzset   000000000006element:000000000004
myzset   000000000007element:000000000004
myzset   000000000009element:000000000004
myzset   000000000000element:000000000006
myzset   000000000005element:000000000009
myzset   000000000005element:000000000006
myzset   000000000002element:000000000006
myzset   000000000003element:000000000004

Leave a message

Name

Email



Send